From 6c404fbc022be0a0d6f1b00983906ad14fe3cc1e Mon Sep 17 00:00:00 2001 From: Mark Payne Date: Tue, 21 Jun 2016 17:34:15 -0400 Subject: [PATCH] NIFI-2065: When a provenance query matches the max number of results requested, stop querying lucene for improved performance --- .../ProvenanceQueryEndpointMerger.java | 10 ++++++++-- .../nifi/web/controller/ControllerFacade.java | 10 ++++++++-- .../js/nf/provenance/nf-provenance-table.js | 4 ++-- .../PersistentProvenanceRepository.java | 4 ---- .../nifi/provenance/lucene/DocsReader.java | 2 +- .../nifi/provenance/lucene/IndexSearch.java | 17 +++++++++++++++++ 6 files changed, 36 insertions(+), 11 deletions(-) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ProvenanceQueryEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ProvenanceQueryEndpointMerger.java index 4875499dbb79..6e0e2b51300f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ProvenanceQueryEndpointMerger.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ProvenanceQueryEndpointMerger.java @@ -176,8 +176,14 @@ public int compare(final ProvenanceEventDTO o1, final ProvenanceEventDTO o2) { results.setErrors(errors); } - results.setTotalCount(totalRecords); - results.setTotal(FormatUtils.formatCount(totalRecords)); + if (clientDto.getRequest().getMaxResults() != null && totalRecords >= clientDto.getRequest().getMaxResults()) { + results.setTotalCount(clientDto.getRequest().getMaxResults().longValue()); + results.setTotal(FormatUtils.formatCount(clientDto.getRequest().getMaxResults().longValue()) + "+"); + } else { + results.setTotal(FormatUtils.formatCount(totalRecords)); + results.setTotalCount(totalRecords); + } + results.setProvenanceEvents(selectedResults); results.setOldestEvent(oldestEventDate); results.setGenerated(new Date()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java index 37ef23743c89..68208ec150ab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java @@ -964,8 +964,14 @@ public ProvenanceDTO getProvenanceQuery(String provenanceId) { events.add(createProvenanceEventDto(record)); } resultsDto.setProvenanceEvents(events); - resultsDto.setTotalCount(queryResult.getTotalHitCount()); - resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount())); + + if (requestDto.getMaxResults() != null && queryResult.getTotalHitCount() >= requestDto.getMaxResults()) { + resultsDto.setTotalCount(requestDto.getMaxResults().longValue()); + resultsDto.setTotal(FormatUtils.formatCount(requestDto.getMaxResults().longValue()) + "+"); + } else { + resultsDto.setTotalCount(queryResult.getTotalHitCount()); + resultsDto.setTotal(FormatUtils.formatCount(queryResult.getTotalHitCount())); + } // include any errors if (queryResult.getError() != null) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js index 4cc9a57718da..e0d70ef19951 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/provenance/nf-provenance-table.js @@ -989,7 +989,7 @@ nf.ProvenanceTable = (function () { // update the filter message based on the request if (isBlankQuery(provenanceRequest)) { var message = 'Showing the most recent '; - if (provenanceResults.totalCount > config.maxResults) { + if (provenanceResults.totalCount >= config.maxResults) { message += (nf.Common.formatInteger(config.maxResults) + ' of ' + provenanceResults.total + ' events, please refine the search.'); } else { message += ('events.'); @@ -998,7 +998,7 @@ nf.ProvenanceTable = (function () { $('#clear-provenance-search').hide(); } else { var message = 'Showing '; - if (provenanceResults.totalCount > config.maxResults) { + if (provenanceResults.totalCount >= config.maxResults) { message += (nf.Common.formatInteger(config.maxResults) + ' of ' + provenanceResults.total + ' events that match the specified query, please refine the search.'); } else { message += ('the events that match the specified query.'); diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java index f91f6a4472fe..b8d6f356cca6 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/PersistentProvenanceRepository.java @@ -2330,10 +2330,6 @@ public void run() { final IndexSearch search = new IndexSearch(PersistentProvenanceRepository.this, indexDir, indexManager, maxAttributeChars); final StandardQueryResult queryResult = search.search(query, retrievalCount, firstEventTimestamp); submission.getResult().update(queryResult.getMatchingEvents(), queryResult.getTotalHitCount()); - if (queryResult.isFinished()) { - logger.info("Successfully executed Query[{}] against Index {}; Search took {} milliseconds; Total Hits = {}", - query, indexDir, queryResult.getQueryTime(), queryResult.getTotalHitCount()); - } } catch (final Throwable t) { logger.error("Failed to query Provenance Repository Index {} due to {}", indexDir, t.toString()); if (logger.isDebugEnabled()) { diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java index 703a5b881882..b195559d6cce 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/DocsReader.java @@ -127,7 +127,7 @@ public Set read(final List docs, final Collecti maxAttributeChars)) { Iterator docIter = byStorageNameDocGroups.get(storageFileName).iterator(); - while (docIter.hasNext() && retrievalCount.incrementAndGet() < maxResults){ + while (docIter.hasNext() && retrievalCount.getAndIncrement() < maxResults) { ProvenanceEventRecord eRec = this.getRecord(docIter.next(), reader); if (eRec != null) { matchingRecords.add(eRec); diff --git a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java index b8661df2fd46..65a06acc562c 100644 --- a/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java +++ b/nifi-nar-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/lucene/IndexSearch.java @@ -49,6 +49,18 @@ public IndexSearch(final PersistentProvenanceRepository repo, final File indexDi } public StandardQueryResult search(final org.apache.nifi.provenance.search.Query provenanceQuery, final AtomicInteger retrievedCount, final long firstEventTimestamp) throws IOException { + if (retrievedCount.get() >= provenanceQuery.getMaxResults()) { + final StandardQueryResult sqr = new StandardQueryResult(provenanceQuery, 1); + sqr.update(Collections. emptyList(), 0L); + + logger.info("Skipping search of Provenance Index {} for {} because the max number of results ({}) has already been retrieved", + indexDirectory, provenanceQuery, provenanceQuery.getMaxResults()); + + return sqr; + } + + final long startNanos = System.nanoTime(); + if (!indexDirectory.exists() && !indexDirectory.mkdirs()) { throw new IOException("Unable to create Indexing Directory " + indexDirectory); } @@ -97,6 +109,11 @@ public StandardQueryResult search(final org.apache.nifi.provenance.search.Query logger.debug("Reading {} records took {} millis for {}", matchingRecords.size(), TimeUnit.NANOSECONDS.toMillis(readRecordsNanos), this); sqr.update(matchingRecords, topDocs.totalHits); + + final long queryNanos = System.nanoTime() - startNanos; + logger.info("Successfully executed {} against Index {}; Search took {} milliseconds; Total Hits = {}", + provenanceQuery, indexDirectory, TimeUnit.NANOSECONDS.toMillis(queryNanos), topDocs.totalHits); + return sqr; } catch (final FileNotFoundException e) { // nothing has been indexed yet, or the data has already aged off