Improve BOM processing performance and make it transactional (#218)

* Add bloated BOM for ingestion performance testing

Signed-off-by: nscuro <nscuro@protonmail.com>

* Prevent query compilation cache being bypassed for `matchSingleIdentity` queries

See DependencyTrack/dependency-track#2540

This also cleans the query from containing weird statements like `(cpe != null && cpe == null)` in case a component does not have a CPE.

Signed-off-by: nscuro <nscuro@protonmail.com>

* WIP: Improve BOM processing performance

Signed-off-by: nscuro <nscuro@protonmail.com>

* Handle dependency graph

Signed-off-by: nscuro <nscuro@protonmail.com>

* Improve dependency graph assembly

Instead of using individual bulk UPDATE queries, use setters on persistent components instead. This way we can again make use of batched flushing.

Signed-off-by: nscuro <nscuro@protonmail.com>

* Completely replace old processing logic

Also decompose large processing method into multiple smaller ones, and re-implement notifications.

Signed-off-by: nscuro <nscuro@protonmail.com>

* Fix not all BOM refs being updated with new component identities

Signed-off-by: nscuro <nscuro@protonmail.com>

* Be smarter about indexing component identities and BOM refs

Also add more documentation

Signed-off-by: nscuro <nscuro@protonmail.com>

* Reduce logging noise

Signed-off-by: nscuro <nscuro@protonmail.com>

* Mark new components as such

... via new transient field. Required for compatibility with #217

Signed-off-by: nscuro <nscuro@protonmail.com>

* Compatibility with #217

Signed-off-by: nscuro <nscuro@protonmail.com>

* Cleanup tests

Signed-off-by: nscuro <nscuro@protonmail.com>

* Reduce code duplication

Signed-off-by: nscuro <nscuro@protonmail.com>

* Cleanup; Process services

Signed-off-by: nscuro <nscuro@protonmail.com>

* Finishing touches 🪄

Signed-off-by: nscuro <nscuro@protonmail.com>

* Make flush threshold configurable

The optimal value could depend on how beefy the database server is, and how much memory is available to the API server.

Signed-off-by: nscuro <nscuro@protonmail.com>

* Clarify `warn` log when rolling back active transactions

Signed-off-by: nscuro <nscuro@protonmail.com>

* Log number of consumed components and services before and after de-dupe

Signed-off-by: nscuro <nscuro@protonmail.com>

* Extend BOM processing test with bloated BOM

Signed-off-by: nscuro <nscuro@protonmail.com>

* Make component identity matching strict

To address DependencyTrack/dependency-track#2519 (comment).

Also add regression test for this specific issue.

Signed-off-by: nscuro <nscuro@protonmail.com>

* Add regression test for DependencyTrack/dependency-track#1905

Signed-off-by: nscuro <nscuro@protonmail.com>

* Clarify why "reachability on commit" is disabled; Add assertion for persistent object state

Signed-off-by: nscuro <nscuro@protonmail.com>

* Add tests for `equals` and `hashCode` of `ComponentIdentity`

Signed-off-by: nscuro <nscuro@protonmail.com>

* Address review comments

Signed-off-by: nscuro <nscuro@protonmail.com>

---------

Signed-off-by: nscuro <nscuro@protonmail.com>
Signed-off-by: mehab <meha.bhargava@citi.com>

src/main/java/org/dependencytrack/common/ConfigKey.java

@@ -2,8 +2,6 @@
 
 import alpine.Config;
 
-import java.time.Duration;
-
 public enum ConfigKey implements Config.Key {
 
     SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true),
@@ -33,7 +31,8 @@ public enum ConfigKey implements Config.Key {
     TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR("task.componentIdentification.lockAtMostForInMillis", "900000"),
     TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR("task.componentIdentification.lockAtLeastForInMillis", "3000"),
     TASK_LDAP_SYNC_LOCK_AT_MOST_FOR("task.ldapSync.lockAtMostForInMillis", "900000"),
-    TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR("task.ldapSync.lockAtLeastForInMillis", "3000");
+    TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR("task.ldapSync.lockAtLeastForInMillis", "3000"),
+    BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD("bom.upload.processing.trx.flush.threshold", "10000");
 
     private final String propertyName;
     private final Object defaultValue;

src/main/java/org/dependencytrack/event/BomUploadEvent.java

@@ -19,9 +19,9 @@
 package org.dependencytrack.event;
 
 import alpine.event.framework.AbstractChainableEvent;
+import org.dependencytrack.model.Project;
 
 import java.io.File;
-import java.util.UUID;
 
 /**
  * Defines an event triggered when a bill-of-material (bom) document is submitted.
@@ -31,16 +31,16 @@
  */
 public class BomUploadEvent extends AbstractChainableEvent {
 
-    private final UUID projectUuid;
+    private final Project project;
     private final File file;
 
-    public BomUploadEvent(final UUID projectUuid, final File file) {
-        this.projectUuid = projectUuid;
+    public BomUploadEvent(final Project project, final File file) {
+        this.project = project;
         this.file = file;
     }
 
-    public UUID getProjectUuid() {
-        return projectUuid;
+    public Project getProject() {
+        return project;
     }
 
     public File getFile() {

src/main/java/org/dependencytrack/model/Component.java

@@ -352,12 +352,12 @@ public enum FetchGroup {
     private UUID uuid;
 
     private transient String bomRef;
+    private transient String licenseId;
     private transient DependencyMetrics metrics;
     private transient RepositoryMetaComponent repositoryMeta;
+    private transient boolean isNew;
     private transient int usedBy;
-
     private transient Set<String> dependencyGraph;
-
     private transient boolean expandDependencyGraph;
 
     public long getId() {
@@ -741,6 +741,14 @@ public void setRepositoryMeta(RepositoryMetaComponent repositoryMeta) {
         this.repositoryMeta = repositoryMeta;
     }
 
+    public boolean isNew() {
+        return isNew;
+    }
+
+    public void setNew(final boolean aNew) {
+        isNew = aNew;
+    }
+
     public Double getLastInheritedRiskScore() {
         return lastInheritedRiskScore;
     }
@@ -757,6 +765,14 @@ public void setBomRef(String bomRef) {
         this.bomRef = bomRef;
     }
 
+    public String getLicenseId() {
+        return licenseId;
+    }
+
+    public void setLicenseId(final String licenseId) {
+        this.licenseId = licenseId;
+    }
+
     public int getUsedBy() {
         return usedBy;
     }

src/main/java/org/dependencytrack/model/ComponentIdentity.java

@@ -23,6 +23,7 @@
 import org.dependencytrack.util.PurlUtil;
 import org.json.JSONObject;
 
+import java.util.Objects;
 import java.util.UUID;
 
 /**
@@ -137,6 +138,19 @@ public UUID getUuid() {
         return uuid;
     }
 
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final ComponentIdentity that = (ComponentIdentity) o;
+        return objectType == that.objectType && Objects.equals(purl, that.purl) && Objects.equals(purlCoordinates, that.purlCoordinates) && Objects.equals(cpe, that.cpe) && Objects.equals(swidTagId, that.swidTagId) && Objects.equals(group, that.group) && Objects.equals(name, that.name) && Objects.equals(version, that.version) && Objects.equals(uuid, that.uuid);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(objectType, purl, purlCoordinates, cpe, swidTagId, group, name, version, uuid);
+    }
+
     public JSONObject toJSON() {
         final JSONObject jsonObject = new JSONObject();
         jsonObject.put("uuid", this.getUuid());