Skip to content

Commit

Permalink
Improve Lucene observability
Browse files Browse the repository at this point in the history 
Collect basic metrics:

* Total number of index operations (`add`, `update`, `delete`, `commit`), grouped by index
* Number of index documents in RAM
* Number of bytes used by the index
* Total number of documents in the index

Also, integrate Lucene's `InfoStream` with Dependency-Track's logging system. Lucene output will now be included when configuring `LOGGING_LEVEL=DEBUG`, or when the respective logger is explicitly configured in `logback.xml`.

Relates to #3429

Signed-off-by: nscuro <nscuro@protonmail.com>
  • Loading branch information
@nscuro
nscuro committed Mar 10, 2024
1 parent f8a6a0c commit ba74b09
Show file tree
Hide file tree
Showing 12 changed files with 456 additions and 320 deletions.
2 changes: 2 additions & 0 deletions docs/_posts/2024-xx-xx-v4.11.0.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ to `true`. Users are highly encouraged to do so.
* Align retry configuration and behavior across analyzers - [apiserver/#3494]
* Add auto-generated changelog to GitHub releases - [apiserver/#3502]
* Bump SPDX license list to v3.23 - [apiserver/#3508]
* Improve observability of Lucene search indexes - [apiserver/#3535]
* Show component count in projects list - [frontend/#683]
* Add current *fail*, *warn*, and *info* values to bottom of policy violation metrics - [frontend/#707]
* Remove unused policy violation widget - [frontend/#710]
Expand Down Expand Up @@ -162,6 +163,7 @@ Special thanks to everyone who contributed code to implement enhancements and fi
[apiserver/#3511]: https://github.com/DependencyTrack/dependency-track/pull/3511
[apiserver/#3512]: https://github.com/DependencyTrack/dependency-track/pull/3512
[apiserver/#3513]: https://github.com/DependencyTrack/dependency-track/pull/3513
[apiserver/#3535]: https://github.com/DependencyTrack/dependency-track/pull/3535

[frontend/#682]: https://github.com/DependencyTrack/frontend/pull/682
[frontend/#683]: https://github.com/DependencyTrack/frontend/pull/683
Expand Down
69 changes: 26 additions & 43 deletions src/main/java/org/dependencytrack/search/ComponentIndexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,14 @@
package org.dependencytrack.search;

import alpine.common.logging.Logger;
import alpine.notification.Notification;
import alpine.notification.NotificationLevel;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.Term;
import org.dependencytrack.model.Component;
import org.dependencytrack.notification.NotificationConstants;
import org.dependencytrack.notification.NotificationGroup;
import org.dependencytrack.notification.NotificationScope;
import org.dependencytrack.persistence.QueryManager;
import org.dependencytrack.search.document.ComponentDocument;

import javax.jdo.Query;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -50,7 +43,7 @@ public final class ComponentIndexer extends IndexManager implements ObjectIndexe
private static final Logger LOGGER = Logger.getLogger(ComponentIndexer.class);
private static final ComponentIndexer INSTANCE = new ComponentIndexer();

protected static ComponentIndexer getInstance() {
static ComponentIndexer getInstance() {
return INSTANCE;
}

Expand All @@ -72,28 +65,15 @@ public String[] getSearchFields() {
* @param component A persisted Component object.
*/
public void add(final ComponentDocument component) {
final Document doc = new Document();
addField(doc, IndexConstants.COMPONENT_UUID, component.uuid().toString(), Field.Store.YES, false);
addField(doc, IndexConstants.COMPONENT_NAME, component.name(), Field.Store.YES, true);
addField(doc, IndexConstants.COMPONENT_GROUP, component.group(), Field.Store.YES, true);
addField(doc, IndexConstants.COMPONENT_VERSION, component.version(), Field.Store.YES, false);
addField(doc, IndexConstants.COMPONENT_SHA1, component.sha1(), Field.Store.YES, true);
addField(doc, IndexConstants.COMPONENT_DESCRIPTION, component.description(), Field.Store.YES, true);
final Document doc = convertToDocument(component);
addDocument(doc);
}

try {
getIndexWriter().addDocument(doc);
} catch (CorruptIndexException e) {
handleCorruptIndexException(e);
} catch (IOException e) {
LOGGER.error("An error occurred while adding component to index", e);
Notification.dispatch(new Notification()
.scope(NotificationScope.SYSTEM)
.group(NotificationGroup.INDEXING_SERVICE)
.title(NotificationConstants.Title.COMPONENT_INDEXER)
.content("An error occurred while adding component to index. Check log for details. " + e.getMessage())
.level(NotificationLevel.ERROR)
);
}
@Override
public void update(ComponentDocument component) {
final Term term = convertToTerm(component);
final Document doc = convertToDocument(component);
updateDocument(term, doc);
}

/**
Expand All @@ -102,20 +82,8 @@ public void add(final ComponentDocument component) {
* @param component A persisted Component object.
*/
public void remove(final ComponentDocument component) {
try {
getIndexWriter().deleteDocuments(new Term(IndexConstants.COMPONENT_UUID, component.uuid().toString()));
} catch (CorruptIndexException e) {
handleCorruptIndexException(e);
} catch (IOException e) {
LOGGER.error("An error occurred while removing a component from the index", e);
Notification.dispatch(new Notification()
.scope(NotificationScope.SYSTEM)
.group(NotificationGroup.INDEXING_SERVICE)
.title(NotificationConstants.Title.COMPONENT_INDEXER)
.content("An error occurred while removing a component from the index. Check log for details. " + e.getMessage())
.level(NotificationLevel.ERROR)
);
}
final Term term = convertToTerm(component);
deleteDocuments(term);
}

/**
Expand Down Expand Up @@ -163,4 +131,19 @@ private static List<ComponentDocument> fetchNext(final QueryManager qm, final Lo
}
}

private Document convertToDocument(final ComponentDocument component) {
final var doc = new Document();
addField(doc, IndexConstants.COMPONENT_UUID, component.uuid().toString(), Field.Store.YES, false);
addField(doc, IndexConstants.COMPONENT_NAME, component.name(), Field.Store.YES, true);
addField(doc, IndexConstants.COMPONENT_GROUP, component.group(), Field.Store.YES, true);
addField(doc, IndexConstants.COMPONENT_VERSION, component.version(), Field.Store.YES, false);
addField(doc, IndexConstants.COMPONENT_SHA1, component.sha1(), Field.Store.YES, true);
addField(doc, IndexConstants.COMPONENT_DESCRIPTION, component.description(), Field.Store.YES, true);
return doc;
}

private static Term convertToTerm(final ComponentDocument component) {
return new Term(IndexConstants.COMPONENT_UUID, component.uuid().toString());
}

}
Loading

0 comments on commit ba74b09

Please sign in to comment.