Skip to content

Commit

Permalink
Priming build and reload improvements.
Browse files Browse the repository at this point in the history
  • Loading branch information
@sdedic
sdedic committed Oct 3, 2023
1 parent f6e7fe3 commit 1f27de7
Show file tree
Hide file tree
Showing 50 changed files with 2,711 additions and 135 deletions.
1 change: 1 addition & 0 deletions ...sp.server/nbcode/integration/src/org/netbeans/modules/nbcode/integration/maven.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,4 @@

primingBuild.snapshot.goals=install
primingBuild.regular.goals=install
skipTests=true
14 changes: 14 additions & 0 deletions java/maven/apichanges.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,20 @@ is the proper place.
<!-- ACTUAL CHANGES BEGIN HERE: -->

<changes>
<change id="partial-maven-project">
<api name="general"/>
<summary>Support for partially loaded projects</summary>
<version major="2" minor="161"/>
<date day="3" month="10" year="2023"/>
<author login="sdedic"/>
<compatibility addition="yes" semantic="compatible"/>
<description>
Added a <a href="@TOP@/org/netbeans/modules/maven/api/NbMavenProject.html#getPartialProject-org.apache.maven.project.MavenProject-">
getPratialProject</a> that returns potentially incompletely loaded project instead of a mocked-up fallback (see <a href="@TOP@/org/netbeans/modules/maven/api/NbMavenProject.html#isErrorPlaceholder-org.apache.maven.project.MavenProject-">isErrorPlaceholder()</a>.
Also added a <a href="@TOP@/org/netbeans/modules/maven/api/NbMavenProject.html#isIncomplete-org.apache.maven.project.MavenProject-">isIncomplete()</a> check that checks project's status.
</description>
<class package="org.netbeans.modules.maven.api" name="NbMavenProject"/>
</change>
<change id="runutils-createconfig-action">
<api name="general"/>
<summary></summary>
Expand Down
3 changes: 2 additions & 1 deletion java/maven/build.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@
<package-artifact-from-dir jarbasename="test-lib3-12.6" relpath="projects/dependencies/repo/grp/test-lib3/12.6"/>
<package-artifact-from-dir jarbasename="test-lib4-12.6" relpath="projects/dependencies/repo/grp/test-lib4/12.6"/>
<package-artifact-from-dir jarbasename="test-processor-12.6" relpath="projects/dependencies/repo/grp/test-processor/12.6"/>


<chmod file="${build.test.unit.dir}/data/projects/multiproject/democa/mvnw" perm="ugo+rwx"/>
</target>

</project>
2 changes: 1 addition & 1 deletion java/maven/nbproject/project.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ javadoc.apichanges=${basedir}/apichanges.xml
javadoc.arch=${basedir}/arch.xml
javahelp.hs=maven.hs
extra.module.files=maven-nblib/
spec.version.base=2.160.0
spec.version.base=2.161.0

# The CPExtender test fails in library processing (not randomly) since NetBeans 8.2; disabling.
test.excludes=**/CPExtenderTest.class
Expand Down
49 changes: 49 additions & 0 deletions java/maven/src/org/netbeans/modules/maven/NbArtifactFixer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.maven.artifact.DefaultArtifact;
Expand Down Expand Up @@ -105,6 +106,10 @@ public class NbArtifactFixer implements ArtifactFixer {
//instead of workarounds down the road, we set the artifact's file here.
// some stacktraces to maven/aether do set it after querying our code, but some don't for reasons unknown to me.
artifact.setFile(f);
Set<Artifact> s = CAPTURE_FAKE_ARTIFACTS.get();
if (s != null) {
s.add(artifact);
}
return f;
} catch (IOException x) {
Exceptions.printStackTrace(x);
Expand Down Expand Up @@ -149,4 +154,48 @@ private static synchronized File createFallbackPOM(String groupId, String artifa
return fallbackPOM;
}

public interface ExceptionCallable<T, E extends Throwable> {
public T call() throws E;
}

@SuppressWarnings("unchecked")
private static <T extends Throwable> void sneakyThrow(Throwable exception) throws T {
throw (T) exception;
}

/**
* Collects faked artifacts, which would be otherwise hidden in maven infrastructure. The value is only valid during {@link #collectFallbackArtifacts}, which
* can be invoked recursively.
*/
private static ThreadLocal<Set<Artifact>> CAPTURE_FAKE_ARTIFACTS = new ThreadLocal<Set<Artifact>>();

/**
* Performs an operation and collects forged artifacts created during that operation. The invocation can be nested; each invocation gets only artifacts from its own 'level',
* not those from possible nested invocations. The function passes on all runtime exceptions and checked exception thrown by the operation.
*
* @param <T> value produced by the executed operation
* @param <E> exception thrown from the operation
* @param code the operation to call and monitor
* @param collector callback that will get collected artifacts.
* @return
* @throws E
*/
public static <T, E extends Throwable> T collectFallbackArtifacts(ExceptionCallable<T, E> code, Consumer<Set<Artifact>> collector) throws E {
Set<Artifact> save = CAPTURE_FAKE_ARTIFACTS.get();
try {
CAPTURE_FAKE_ARTIFACTS.set(new HashSet<>());
return code.call();
} catch (Error | RuntimeException r) {
throw r;
} catch (Exception ex) {
sneakyThrow(ex);
// unreachable
throw new Error();
} finally {
if (collector != null) {
collector.accept(CAPTURE_FAKE_ARTIFACTS.get());
}
CAPTURE_FAKE_ARTIFACTS.set(save);
}
}
}
153 changes: 140 additions & 13 deletions java/maven/src/org/netbeans/modules/maven/NbMavenProjectImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.WeakHashMap;
Expand Down Expand Up @@ -134,6 +133,14 @@ public final class NbMavenProjectImpl implements Project {
private final RequestProcessor.Task reloadTask = RELOAD_RP.create(new Runnable() {
@Override
public void run() {
if (LOG.isLoggable(Level.FINE)) {
MavenProject x;
synchronized (NbMavenProjectImpl.this) {
x = project == null ? null : project.get();
}
LOG.log(Level.FINE, "Project {0} starting reload. Currentproject is: {1}",
new Object[] { System.identityHashCode(x == null ? this : x ) });
}
problemReporter.clearReports(); //#167741 -this will trigger node refresh?
MavenProject prj = loadOriginalMavenProject(true);
MavenProject old;
Expand Down Expand Up @@ -303,6 +310,15 @@ public String getHintJavaPlatform() {
*/
//TODO revisit usage, eventually should be only reuse MavenProjectCache
public @NonNull MavenProject loadMavenProject(MavenEmbedder embedder, List<String> activeProfiles, Properties properties) {
ProjectActionContext.Builder b = ProjectActionContext.newBuilder(this).
withProfiles(activeProfiles);
for (String pn : properties.stringPropertyNames()) {
b.withProperty(pn, properties.getProperty(pn));
}

return MavenProjectCache.loadMavenProject(projectFile,
b.context(), null);
/*
try {
MavenExecutionRequest req = embedder.createMavenExecutionRequest();
req.addActiveProfiles(activeProfiles);
Expand Down Expand Up @@ -340,6 +356,7 @@ public String getHintJavaPlatform() {
LOG.log(Level.INFO, "Runtime exception thrown while loading maven project at " + getProjectDirectory(), exc); //NOI18N
}
return MavenProjectCache.getFallbackProject(this.getPOMFile());
*/
}

/**
Expand Down Expand Up @@ -374,7 +391,8 @@ public MavenProject loadParentOf(MavenEmbedder embedder, MavenProject project) t
request.setRepositorySession(maven.newRepositorySession(req));

if (project.getParentFile() != null) {
parent = builder.build(project.getParentFile(), request).getProject();
req.setPom(project.getParentFile());
parent = MavenProjectCache.loadOriginalMavenProjectInternal(embedder, req);
} else if (project.getModel().getParent() != null) {
parent = builder.build(project.getParentArtifact(), request).getProject();
}
Expand Down Expand Up @@ -408,6 +426,21 @@ public Map<? extends String,? extends String> createSystemPropsForPropertyExpres
public Map<? extends String,? extends String> createUserPropsForPropertyExpressions() {
return NbCollections.checkedMapByCopy(configProvider.getActiveConfiguration().getProperties(), String.class, String.class, true);
}

/**
* Returns the current parsed project state. May return {@code null}, if the project was never loaded or expired from the cache, but
* never blocks on Maven infrastructure and is very fast.
* @return current project or {@code null}
*/
@CheckForNull
public MavenProject getOriginalMavenProjectOrNull() {
synchronized (this) {
if (project == null) {
return null;
}
return project.get();
}
}

/**
* getter for the maven's own project representation.. this instance is cached but gets reloaded
Expand Down Expand Up @@ -435,7 +468,8 @@ public Map<? extends String,? extends String> createUserPropsForPropertyExpress
}

/**
* Returns the original project, or waits for reload task if already pending.
* Returns the original project, or waits for reload task if already pending. Use with care, as
* the method blocks until the project reload eventually finishes in the reload thread / RP.
* @return possibly reloaded Maven project.
*/
public MavenProject getFreshOriginalMavenProject() {
Expand Down Expand Up @@ -544,12 +578,6 @@ void stopHardReferencingMavenPoject() {
model = null;
}
newproject = MavenProjectCache.getMavenProject(this.getPOMFile(), reload);
if (newproject == null) { //null when no pom.xml in project folder..
newproject = MavenProjectCache.getFallbackProject(projectFile);
LOG.log(Level.FINE, "Project {0} going to fallback, with packaging: {1}", new Object[] { getPOMFile(), newproject.getPackaging() });
}
final MavenExecutionResult res = MavenProjectCache.getExecutionResult(newproject);
final MavenProject np = newproject;
} finally {
if (LOG.isLoggable(Level.FINE) && SwingUtilities.isEventDispatchThread()) {
LOG.log(Level.FINE, "Project " + getProjectDirectory().getPath() + " loaded in AWT event dispatching thread!", new RuntimeException());
Expand All @@ -559,29 +587,123 @@ void stopHardReferencingMavenPoject() {
return newproject;
}

/**
* Task that potential project reloads should wait on. If set, a {@link fireProjectReload}(true) will be scheduled only after this blocker finishes.
*/
private RequestProcessor.Task reloadBlocker;


/**
* Schedules project operation that delays potential reloads. If a reload is posted, it will be performed only after
* this operation compeltes (successfully, or erroneously). Multiple project operations can be scheduled, an eventual project reload
* should happen after all those operations complete. It is possible to postpone project reload indefinitely, avoid unnecessary
* operation schedules.
*
* @param r operation to run
* @return controlling task.
*/
public RequestProcessor.Task scheduleProjectOperation(RequestProcessor.Task t) {
if (Boolean.getBoolean("test.reload.sync")) {
LOG.log(Level.FINE, "Running the blocking task synchronously (test.reload.sync set)");
t.run();
return t;
} else {
synchronized (this) {
if (reloadBlocker == null) {
LOG.log(Level.FINER, "Blocking project reload on task {0}", t);
reloadBlocker = t;
return t;
} else {
// will chain after existing reload blocker AND the new task.
final RequestProcessor.Task t2 = RELOAD_RP.create(() -> {});
LOG.log(Level.FINER, "Creating project blocker {0}, chain after existing blocker {1}", new Object[] { t2, t });
reloadBlocker.addTaskListener((e) -> {
t.addTaskListener((e2) -> {
synchronized (NbMavenProjectImpl.this) {
if (t2 == reloadBlocker) {
reloadBlocker = null;
}
}
t2.run();
});
});
reloadBlocker = t2;
return t2;
}
}
}
}

public RequestProcessor.Task fireProjectReload() {
return fireProjectReload(false);
}

/**
* Schedules project reload. If `waitForBlockers` is true and {@link #scheduleProjectOperation} registered some task(s), project reload
* will be postponed until after those task(s) finish. The returned task completes after the project reload itself completes (after the potential
* delays).
* <p>
* As a result of project's reload, child projects may be reloaded, but the returned task does not wait for children reload to complete.
*
* @param waitForBlockers
* @return the task that completes after project reloads.
*/
public RequestProcessor.Task fireProjectReload(boolean waitForBlockers) {
//#227101 not only AWT and project read/write mutex has to be checked, there are some additional more
//complex scenarios that can lead to deadlock. Just give up and always fire changes in separate RP.
if (Boolean.getBoolean("test.reload.sync")) {
reloadTask.run();
//for tests just do sync reload, even though silly, even sillier is to attempt to sync the threads..
} else {
RequestProcessor.Task t;
synchronized (this) {
if (reloadBlocker != null && waitForBlockers) {
// avoid holding the lock while potentially executing the listener in-line
t = reloadBlocker;
} else {
t = null;
}
}
if (t != null) {
final RequestProcessor.Task t2 = RELOAD_RP.create(() -> {});
LOG.log(Level.FINER, "Scheduling project reload retry after blocker {0} completes, Returning task {1}", new Object[] { t, t2 });
t.addTaskListener((e) -> {
synchronized (NbMavenProjectImpl.this) {
if (t == reloadBlocker) {
reloadBlocker = null;
}
}
fireProjectReload(true).addTaskListener((e2) -> {
// mark the t2 task finished.
if (!t2.isFinished()) {
t2.run();
}
});
});
return t2;
}
LOG.log(Level.FINER, "Scheduling prject reload {0}, no blocker found.", this);
LOG.log(Level.FINER, "Stack trace:", new Throwable());
reloadTask.schedule(0); //asuming here that schedule(0) will move the scheduled task in the queue if not yet executed
}
return reloadTask;
}

private void reloadPossibleBrokenModules(MavenProject preceding, MavenProject p) {
LOG.log(Level.FINE, "Recovery for project {2}, preceding: {0}, current: {1}, ",
new Object[] { preceding == null ? -1 : System.identityHashCode(preceding), System.identityHashCode(p), p });
// restrict to just poms that were marked as broken/incomplete.
if (!(MavenProjectCache.isFallbackproject(preceding) ||
if (!(MavenProjectCache.isIncompleteProject(preceding) ||
// the project is tagged by Boolean.TRUE, if a SanityBuildAction was created for it.
preceding.getContextValue("org.netbeans.modules.maven.problems.primingNotDone") == Boolean.TRUE)) {
LOG.log(Level.FINER, "Project is not fallbach: {0}, {1}", new Object[] {
MavenProjectCache.isIncompleteProject(preceding),
preceding.getContextValue("org.netbeans.modules.maven.problems.primingNotDone")
});
return;
}
// but do not cascade from projects, which are themselves broken.
if (MavenProjectCache.isFallbackproject(p)) {
LOG.log(Level.FINE, "New project is still fallback, skipping");
return;
}
File basePOMFile = p.getFile().getParentFile();
Expand All @@ -593,10 +715,15 @@ private void reloadPossibleBrokenModules(MavenProject preceding, MavenProject p)
}
MavenProject child = MavenProjectCache.getMavenProject(modPom, true, false);
if (child == null) {
LOG.log(Level.FINE, "Child project {0} is not cached yet", modPom);
continue;
}
LOG.log(Level.FINE, "Child project fallback status: {0}, {1}", new Object[] {
MavenProjectCache.isIncompleteProject(child),
child.getContextValue("org.netbeans.modules.maven.problems.primingNotDone")
});
// the project may have more problems, more subtle, but now repair just total breakage
if (!MavenProjectCache.isFallbackproject(child) && child.getContextValue("org.netbeans.modules.maven.problems.primingNotDone") != Boolean.TRUE) {
if (!MavenProjectCache.isIncompleteProject(child) && child.getContextValue("org.netbeans.modules.maven.problems.primingNotDone") != Boolean.TRUE) {
LOG.log(Level.FINE, "Project for module {0} is not a fallback, skipping", modName);
continue;
}
Expand All @@ -612,7 +739,7 @@ private void reloadPossibleBrokenModules(MavenProject preceding, MavenProject p)
} else {
LOG.log(Level.INFO, "Recovering module {0}, pomfile {1}", new Object[] { modName, modPom });
NbMavenProjectImpl childImpl = c.getLookup().lookup(NbMavenProjectImpl.class);
childImpl.fireProjectReload();
childImpl.fireProjectReload(true);
}
} catch (IOException ex) {
LOG.log(Level.FINE, "Error getting module project {0} is not a project", modName);
Expand Down

0 comments on commit 1f27de7

Please sign in to comment.