Skip to content

Commit

Permalink
#1026 - Refactored how component iteration is handled throughout DT. …
Browse files Browse the repository at this point in the history 
…Added logic to break out of iterations when component inventory changes. This should eliminate the possibility of endless loops. Added additional logging during iterations.
  • Loading branch information
@stevespringett
stevespringett committed Jun 9, 2021
1 parent a9307c7 commit 6d442a5
Show file tree
Hide file tree
Showing 10 changed files with 129 additions and 174 deletions.
33 changes: 25 additions & 8 deletions src/main/java/org/dependencytrack/search/ComponentIndexer.java
View file
Expand Up @@ -22,10 +22,14 @@
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import alpine.persistence.PaginatedResult;
import alpine.resources.AlpineRequest;
import alpine.resources.OrderDirection;
import alpine.resources.Pagination;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.Project;
import org.dependencytrack.notification.NotificationConstants;
import org.dependencytrack.notification.NotificationGroup;
import org.dependencytrack.notification.NotificationScope;
Expand Down Expand Up @@ -115,16 +119,29 @@ public void remove(final Component component) {
public void reindex() {
LOGGER.info("Starting reindex task. This may take some time.");
super.reindex();
try (QueryManager qm = new QueryManager()) {
final long total = qm.getCount(Component.class);
final AlpineRequest alpineRequest = new AlpineRequest(
null,
new Pagination(Pagination.Strategy.OFFSET, 0, 100),
null,
"id",
OrderDirection.ASCENDING
);
try (final QueryManager qm = new QueryManager(alpineRequest)) {
final PaginatedResult result = qm.getProjects(false, true);
long count = 0;
while (count < total) {
final PaginatedResult result = qm.getComponents();
final List<Component> components = result.getList(Component.class);
for (final Component component: components) {
add(component);
boolean shouldContinue = true;
while (count < result.getTotal() && shouldContinue) {
for (final Project project: result.getList(Project.class)) {
final List<Component> components = qm.getAllComponents(project);
LOGGER.info("Indexing " + components.size() + " components in project: " + project.getUuid());
for (final Component component: components) {
add(component);
}
LOGGER.info("Completed indexing of " + components.size() + " components in project: " + project.getUuid());
}
count += result.getObjects().size();
int lastResult = result.getObjects().size();
count += lastResult;
shouldContinue = lastResult > 0;
qm.advancePagination();
}
commit();
Expand Down
16 changes: 9 additions & 7 deletions src/main/java/org/dependencytrack/search/ProjectIndexer.java
View file
Expand Up @@ -25,6 +25,7 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.Project;
import org.dependencytrack.notification.NotificationConstants;
import org.dependencytrack.notification.NotificationGroup;
Expand Down Expand Up @@ -126,16 +127,17 @@ public void remove(final Project project) {
public void reindex() {
LOGGER.info("Starting reindex task. This may take some time.");
super.reindex();
try (QueryManager qm = new QueryManager()) {
final long total = qm.getCount(Project.class);
try (final QueryManager qm = new QueryManager()) {
final PaginatedResult result = qm.getProjects(false, true);
long count = 0;
while (count < total) {
final PaginatedResult result = qm.getProjects();
final List<Project> projects = result.getList(Project.class);
for (final Project project: projects) {
boolean shouldContinue = true;
while (count < result.getTotal() && shouldContinue) {
for (final Project project: result.getList(Project.class)) {
add(project);
}
count += result.getObjects().size();
int lastResult = result.getObjects().size();
count += lastResult;
shouldContinue = lastResult > 0;
qm.advancePagination();
}
commit();
Expand Down
132 changes: 72 additions & 60 deletions src/main/java/org/dependencytrack/tasks/VulnerabilityAnalysisTask.java
View file
Expand Up @@ -21,7 +21,10 @@
import alpine.event.framework.Event;
import alpine.event.framework.Subscriber;
import alpine.logging.Logger;
import com.github.packageurl.PackageURL;
import alpine.persistence.PaginatedResult;
import alpine.resources.AlpineRequest;
import alpine.resources.OrderDirection;
import alpine.resources.Pagination;
import org.apache.commons.collections4.CollectionUtils;
import org.dependencytrack.event.InternalAnalysisEvent;
import org.dependencytrack.event.MetricsUpdateEvent;
Expand All @@ -30,10 +33,9 @@
import org.dependencytrack.event.VulnDbAnalysisEvent;
import org.dependencytrack.event.VulnerabilityAnalysisEvent;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.Project;
import org.dependencytrack.persistence.QueryManager;
import org.dependencytrack.policy.PolicyEngine;
import org.dependencytrack.tasks.scanners.AnalyzerIdentity;
import org.dependencytrack.tasks.scanners.BaseComponentAnalyzerTask;
import org.dependencytrack.tasks.scanners.CacheableScanTask;
import org.dependencytrack.tasks.scanners.InternalAnalysisTask;
import org.dependencytrack.tasks.scanners.NpmAuditAnalysisTask;
Expand All @@ -43,7 +45,7 @@
import java.util.ArrayList;
import java.util.List;

public class VulnerabilityAnalysisTask extends BaseComponentAnalyzerTask implements Subscriber {
public class VulnerabilityAnalysisTask implements Subscriber {

private static final Logger LOGGER = Logger.getLogger(VulnerabilityAnalysisTask.class);

Expand All @@ -52,73 +54,90 @@ public class VulnerabilityAnalysisTask extends BaseComponentAnalyzerTask impleme
private final List<Component> ossIndexCandidates = new ArrayList<>();
private final List<Component> vulnDbCandidates = new ArrayList<>();

public boolean isCapable(final PackageURL purl) {
return true;
}

public AnalyzerIdentity getAnalyzerIdentity() {
return AnalyzerIdentity.NONE;
}

/**
* {@inheritDoc}
*/
public void inform(final Event e) {
if (e instanceof VulnerabilityAnalysisEvent) {
final VulnerabilityAnalysisEvent event = (VulnerabilityAnalysisEvent)e;
final VulnerabilityAnalysisEvent event = (VulnerabilityAnalysisEvent) e;

if (event.getComponents() != null && event.getComponents().size() > 0) {
final List<Component> components = new ArrayList<>();
try (QueryManager qm = new QueryManager()) {
for (Component c : event.getComponents()) {
try (final QueryManager qm = new QueryManager()) {
for (final Component c : event.getComponents()) {
// Ensures the current component (and related objects such as Project) are attached to the
// current persistence manager. This may cause duplicate projects to be created and other
// unexpected behavior.
components.add(qm.getObjectById(Component.class, c.getId()));
}

/*
When this task is processing events that specify the components to scan,
separate them out into 'candidates' so that we can fire off multiple events
in hopes of perform parallel analysis using different analyzers.
*/
final InternalAnalysisTask internalAnalysisTask = new InternalAnalysisTask();
final NpmAuditAnalysisTask npmAuditAnalysisTask = new NpmAuditAnalysisTask();
final OssIndexAnalysisTask ossIndexAnalysisTask = new OssIndexAnalysisTask();
final VulnDbAnalysisTask vulnDbAnalysisTask = new VulnDbAnalysisTask();
for (final Component component : components) {
inspectComponentReadiness(component, internalAnalysisTask, internalCandidates);
inspectComponentReadiness(component, npmAuditAnalysisTask, npmCandidates);
inspectComponentReadiness(component, ossIndexAnalysisTask, ossIndexCandidates);
inspectComponentReadiness(component, vulnDbAnalysisTask, vulnDbCandidates);
analyzeComponents(qm, components);
}
performPolicyEvaluation(event.getProject(), components);
} else {
LOGGER.info("Analyzing portfolio");
final AlpineRequest alpineRequest = new AlpineRequest(
null,
new Pagination(Pagination.Strategy.OFFSET, 0, 100),
null,
"id",
OrderDirection.ASCENDING
);
try (final QueryManager qm = new QueryManager(alpineRequest)) {
final PaginatedResult result = qm.getProjects(false, true);
long count = 0;
boolean shouldContinue = true;
while (count < result.getTotal() && shouldContinue) {
for (final Project project: result.getList(Project.class)) {
final List<Component> components = qm.getAllComponents(project);
LOGGER.info("Analyzing " + components.size() + " components in project: " + project.getUuid());
analyzeComponents(qm, components);
LOGGER.info("Completed analysis of " + components.size() + " components in project: " + project.getUuid());
}
int lastResult = result.getObjects().size();
count += lastResult;
shouldContinue = lastResult > 0;
qm.advancePagination();
}

qm.detach(components);

// Do not call individual async events when processing a known list of components.
// Call each analyzer task sequentially and catch any exceptions as to prevent one analyzer
// from interrupting the successful execution of all analyzers.
performAnalysis(internalAnalysisTask, new InternalAnalysisEvent(internalCandidates));
performAnalysis(npmAuditAnalysisTask, new NpmAuditAnalysisEvent(npmCandidates));
performAnalysis(ossIndexAnalysisTask, new OssIndexAnalysisEvent(ossIndexCandidates));
performAnalysis(vulnDbAnalysisTask, new VulnDbAnalysisEvent(vulnDbCandidates));
}
LOGGER.info("Portfolio analysis complete");
}
}
}

// Evaluate the components against applicable policies via the PolicyEngine.
final PolicyEngine pe = new PolicyEngine();
pe.evaluate(components);
private void analyzeComponents(final QueryManager qm, final List<Component> components) {
/*
When this task is processing events that specify the components to scan,
separate them out into 'candidates' so that we can fire off multiple events
in hopes of perform parallel analysis using different analyzers.
*/
final InternalAnalysisTask internalAnalysisTask = new InternalAnalysisTask();
final NpmAuditAnalysisTask npmAuditAnalysisTask = new NpmAuditAnalysisTask();
final OssIndexAnalysisTask ossIndexAnalysisTask = new OssIndexAnalysisTask();
final VulnDbAnalysisTask vulnDbAnalysisTask = new VulnDbAnalysisTask();
for (final Component component : components) {
inspectComponentReadiness(component, internalAnalysisTask, internalCandidates);
inspectComponentReadiness(component, npmAuditAnalysisTask, npmCandidates);
inspectComponentReadiness(component, ossIndexAnalysisTask, ossIndexCandidates);
inspectComponentReadiness(component, vulnDbAnalysisTask, vulnDbCandidates);
}

if (event.getProject() != null) {
Event.dispatch(new MetricsUpdateEvent(event.getProject()));
}
qm.detach(components);

} else {
// Portfolio analysis\\\
Event.dispatch(new InternalAnalysisEvent());
Event.dispatch(new NpmAuditAnalysisEvent());
Event.dispatch(new OssIndexAnalysisEvent());
Event.dispatch(new VulnDbAnalysisEvent());
}
// Do not call individual async events when processing a known list of components.
// Call each analyzer task sequentially and catch any exceptions as to prevent one analyzer
// from interrupting the successful execution of all analyzers.
performAnalysis(internalAnalysisTask, new InternalAnalysisEvent(internalCandidates));
performAnalysis(npmAuditAnalysisTask, new NpmAuditAnalysisEvent(npmCandidates));
performAnalysis(ossIndexAnalysisTask, new OssIndexAnalysisEvent(ossIndexCandidates));
performAnalysis(vulnDbAnalysisTask, new VulnDbAnalysisEvent(vulnDbCandidates));
}

private void performPolicyEvaluation(Project project, List<Component> components) {
// Evaluate the components against applicable policies via the PolicyEngine.
final PolicyEngine pe = new PolicyEngine();
pe.evaluate(components);
if (project != null) {
Event.dispatch(new MetricsUpdateEvent(project));
}
}

Expand Down Expand Up @@ -152,11 +171,4 @@ private void performAnalysis(final Subscriber scanTask, final VulnerabilityAnaly
event.getComponents().forEach(c -> c.setCacheResult(null));
}
}

/**
* Unused.
*/
public void analyze(final List<Component> components) {
}

}
26 changes: 16 additions & 10 deletions src/main/java/org/dependencytrack/tasks/repositories/RepositoryMetaAnalyzerTask.java
View file
Expand Up @@ -28,6 +28,7 @@
import org.apache.commons.lang3.StringUtils;
import org.dependencytrack.event.RepositoryMetaEvent;
import org.dependencytrack.model.Component;
import org.dependencytrack.model.Project;
import org.dependencytrack.model.Repository;
import org.dependencytrack.model.RepositoryMetaComponent;
import org.dependencytrack.persistence.QueryManager;
Expand All @@ -54,22 +55,27 @@ public void inform(final Event e) {
} else {
final AlpineRequest alpineRequest = new AlpineRequest(
null,
new Pagination(Pagination.Strategy.OFFSET, 0, 1000),
new Pagination(Pagination.Strategy.OFFSET, 0, 100),
null,
"id",
OrderDirection.ASCENDING
);
try (QueryManager qm = new QueryManager(alpineRequest)) {
final long total = qm.getCount(Component.class);
LOGGER.info("Performing component repository metadata analysis against all (" + total +") components in the portfolio");
try (final QueryManager qm = new QueryManager(alpineRequest)) {
final PaginatedResult result = qm.getProjects(false, true);
long count = 0;
while (count < total) {
final PaginatedResult result = qm.getComponents();
final List<Component> components = result.getList(Component.class);
for (final Component component: components) {
analyze(qm, component);
boolean shouldContinue = true;
while (count < result.getTotal() && shouldContinue) {
for (final Project project: result.getList(Project.class)) {
final List<Component> components = qm.getAllComponents(project);
LOGGER.info("Performing component repository metadata analysis against " + components.size() + " components in project: " + project.getUuid());
for (final Component component: components) {
analyze(qm, component);
}
LOGGER.info("Completed component repository metadata analysis against " + components.size() + " components in project: " + project.getUuid());
}
count += result.getObjects().size();
int lastResult = result.getObjects().size();
count += lastResult;
shouldContinue = lastResult > 0;
qm.advancePagination();
}
}
Expand Down

0 comments on commit 6d442a5

Please sign in to comment.