NIFI-12855: Add more information to provenance events to facilitate f…

…ull graph traversal

Generate event IDs earlier so previousEventIds can be updated accurately

Added flow graph components to provenance graph

Fix component nodes to update properties rather than a new node

Add capabilities (some placeholders) for creating the database and indexes, added relationship property to 'outgoing' provenance events

Added relationships to all provenance events except clone and join, also some Cypher improvements

Added some fine-grained exceptions for better error handling/reporting

Split query generation API from GraphClientService into finer-grained methods

Replace Collections.emptyList() with ArrayList<>[0] for modifiability

Fix copyright headers

minifi/minifi-nar-bundles/minifi-provenance-repository-bundle/minifi-provenance-repositories/src/main/java/org/apache/nifi/provenance/NoOpProvenanceRepository.java

@@ -42,8 +42,8 @@ public class NoOpProvenanceRepository implements ProvenanceRepository {
 
   @Override
   public void initialize(EventReporter eventReporter, Authorizer authorizer,
-      ProvenanceAuthorizableFactory factory, IdentifierLookup identifierLookup)
-      throws IOException {
+                         ProvenanceAuthorizableFactory factory, IdentifierLookup identifierLookup)
+          throws IOException {
 
   }
 
@@ -69,13 +69,13 @@ public ProvenanceEventRecord getEvent(final long id, final NiFiUser user) throws
 
   @Override
   public List<ProvenanceEventRecord> getEvents(long firstRecordId, int maxRecords)
-      throws IOException {
+          throws IOException {
     return emptyList();
   }
 
   @Override
   public List<ProvenanceEventRecord> getEvents(long firstRecordId,
-      int maxRecords, NiFiUser niFiUser) throws IOException {
+                                               int maxRecords, NiFiUser niFiUser) throws IOException {
     return emptyList();
   }
 
@@ -156,7 +156,7 @@ public long getContainerUsableSpace(String s) throws IOException {
 
   @Override
   public AsyncLineageSubmission retrieveLineageSubmission(final String lineageIdentifier,
-      final NiFiUser user) {
+                                                          final NiFiUser user) {
     return null;
   }
 

nifi-api/src/main/java/org/apache/nifi/provenance/ProvenanceEventRecord.java

@@ -40,8 +40,6 @@ public interface ProvenanceEventRecord {
      */
     List<Long> getPreviousEventIds();
 
-    void setPreviousEventIds(List<Long> previousEventIds);
-
     /**
      * @return the time at which this Provenance Event was created, as the
      * number of milliseconds since epoch

nifi-nar-bundles/nifi-graph-bundle/nifi-graph-client-service-api/src/main/java/org/apache/nifi/graph/TinkerPopClientService.java → nifi-api/src/main/java/org/apache/nifi/provenance/UpdateableProvenanceEventRecord.java

@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.nifi.provenance;
 
-package org.apache.nifi.graph;
+import java.util.List;
 
-/**
- * This interface is meant to make it easier for anyone who wants to write components to target TinkerPop/OpenCypher
- * client services only.
- */
-public interface TinkerPopClientService extends GraphClientService {
+public interface UpdateableProvenanceEventRecord extends ProvenanceEventRecord {
+
+    void setEventId(final long eventId);
+    void setPreviousEventIds(List<Long> previousEventIds);
 }

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/PlaceholderProvenanceEvent.java

@@ -25,9 +25,9 @@
  * A Provenance Event that is used to replace another Provenance Event when authorizations
  * are not granted for the original Provenance Event
  */
-public class PlaceholderProvenanceEvent implements ProvenanceEventRecord {
+public class PlaceholderProvenanceEvent implements UpdateableProvenanceEventRecord {
     private final String componentId;
-    private final long eventId;
+    private long eventId;
     private List<Long> previousEventIds;
     private final long eventTime;
     private final String flowFileUuid;
@@ -45,6 +45,11 @@ public long getEventId() {
         return eventId;
     }
 
+    @Override
+    public void setEventId(long eventId) {
+        this.eventId = eventId;
+    }
+
     @Override
     public List<Long> getPreviousEventIds() {
         return previousEventIds;

nifi-commons/nifi-data-provenance-utils/src/main/java/org/apache/nifi/provenance/StandardProvenanceEventRecord.java

@@ -30,7 +30,7 @@
 /**
  * Holder for provenance relevant information
  */
-public class StandardProvenanceEventRecord implements ProvenanceEventRecord {
+public class StandardProvenanceEventRecord implements UpdateableProvenanceEventRecord {
 
     private final long eventTime;
     private final long entryDate;
@@ -125,7 +125,8 @@ public long getStorageByteOffset() {
         return storageByteOffset;
     }
 
-    void setEventId(final long eventId) {
+    @Override
+    public void setEventId(final long eventId) {
         this.eventId = eventId;
     }
 
@@ -316,7 +317,8 @@ public int hashCode() {
         }
 
         return -37423 + 3 * componentId.hashCode() + (transitUri == null ? 0 : 41 * transitUri.hashCode())
-                + (relationship == null ? 0 : 47 * relationship.hashCode()) + 44 * eventTypeCode
+                //+ (relationship == null ? 0 : 47 * relationship.hashCode())
+                + 44 * eventTypeCode
                 + 47 * getChildUuids().hashCode() + 47 * getParentUuids().hashCode();
     }
 
@@ -363,10 +365,6 @@ public boolean equals(final Object obj) {
             return false;
         }
 
-        if (different(relationship, other.relationship)) {
-            return false;
-        }
-
         return !(eventType == ProvenanceEventType.REPLAY && eventTime != other.getEventTime());
     }
 
@@ -431,7 +429,7 @@ public String toString() {
                 + ", uuid=" + uuid
                 + ", fileSize=" + contentSize
                 + ", componentId=" + componentId
-                + ", componentType" + componentType
+                + ", componentType=" + componentType
                 + ", transitUri=" + transitUri
                 + ", sourceSystemFlowFileIdentifier=" + sourceSystemFlowFileIdentifier
                 + ", parentUuids=" + parentUuids
@@ -535,7 +533,6 @@ public Builder fromEvent(final ProvenanceEventRecord event) {
             }
 
             previousEventIds = event.getPreviousEventIds();
-
             return this;
         }