Skip to content

Commit

Permalink
Merge pull request #13 from polarityio/develop
Browse files Browse the repository at this point in the history 
INT-995:  Support paging of results
  • Loading branch information
@sarus
sarus committed Jun 27, 2023
2 parents b1da813 + 0de00a6 commit 61779ea
Show file tree
Hide file tree
Showing 9 changed files with 187 additions and 34 deletions.
18 changes: 13 additions & 5 deletions README.md
View file
Expand Up @@ -53,30 +53,38 @@ kibana_sample_data_logs,kibana_sample_data_flights

The search query to execute as JSON. The top level property should be a `query` object and must be a valid JSON search request when sent to the ES `_search` REST endpoint. The search query can make use of the templated variable `{{entity}}` which will be replaced by the entity recognized on the user's screen.

You should not specify `size` and `from` parameters as these parameters are controlled with the "Page Size" option.

As an example, with the search query defined as:

```
{"query": { "simple_query_string": { "query": "\"{{entity}}\"" } }, "from": 0, "size": 10, "sort": [ {"timestamp": "desc" } ] } }
{"query": { "simple_query_string": { "query": "\"{{entity}}\"" } }, "sort": [ {"timestamp": "desc" } ] } }
```

If the user has the IP 8.8.8.8 on their screen the integration will execute the following query:

```
{"query": { "simple_query_string": { "query": "\"8.8.8.8\"" } }, "from": 0, "size": 10, "sort": [ {"timestamp": "desc" } ] } }
{"query": { "simple_query_string": { "query": "\"8.8.8.8\"" } }, "sort": [ {"timestamp": "desc" } ] } }
```

If you'd like to search certain fields you can use the `fields` property along with the `simple_query_string`. For example, to only search the `ip` field you could use the following search:

```
{"query": { "simple_query_string": { "query": "\"{{entity}}\"", "fields": ["ip"]}}, "from": 0, "size": 10, "sort": [ {"timestamp": "desc" } ] } }
{"query": { "simple_query_string": { "query": "\"{{entity}}\"", "fields": ["ip"]}}, "sort": [ {"timestamp": "desc" } ] } }
```

If you'd like to search a specific time range you can do that using normal Elasticsearch JSON search syntax. For example, do search data from the last 365 days you can use the following query assuming your timestamp field is called `timestamp`.

```
{"query": { "bool": { "must": [ { "range": {"timestamp": {"gte": "now-365d/d","lt": "now/d"}}},{"query_string": {"query":"\"{{entity}}\""}}]}},"from": 0,"size": 10}
{"query": { "bool": { "must": [ { "range": {"timestamp": {"gte": "now-365d/d","lt": "now/d"}}},{"query_string": {"query":"\"{{entity}}\""}}]}}}
```

### Page Size

The number of results to display per page. This value must be between 1 and 100. Defaults to 10. This option should be set to "Only admins can view and edit".

> **Important** The Page Size option is used to set the `size` and `from` search parameters in the query. If you set these values directly in your query, they will be overridden by the Page Size option. As an example, if you set the Page Size to 10, then `size` will be set to 10, and `from` will be set to 0.
### Enable Highlighting

If checked, the integration will display highlighted search terms via the Elasticsearch Highlighter. For more information on the Elasticsearch Highlighter please see the following documentation: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-highlighting.html
Expand All @@ -87,7 +95,7 @@ If checked, the integration will display highlighted search terms via the Elasti
### Highlight Query

The highlighter query to execute when a user clicks to view additional details. The top level property should be a `query` object. This query should typically match the query portion of your `Search Query`. Highlighting will attempt to highlight against all fields and will return the first 10 results. Only runs if the `Enable Highlighting` option is checked
The highlighter query to execute when a user clicks to view additional details. The top level property should be a `query` object. This query should typically match the query portion of your `Search Query`. Highlighting will attempt to highlight against all fields. Only runs if the `Enable Highlighting` option is checked

```
{"query": { "simple_query_string": { "query": "\"{{entity}}\"" } } }
Expand Down
68 changes: 56 additions & 12 deletions component/es.js
View file
Expand Up @@ -21,12 +21,11 @@ polarity.export = PolarityComponent.extend({
this.set('block._state', {});
this.set('block._state.searchRunning', false);
this.set('block._state.highlightsLoading', false);
this.set('block._state.paging', {});
this.updatePageParameters();
}

if (
!this.get('details.highlights') &&
!this.get('isSearchLimitReached')
) {
if (!this.get('details.highlights') && !this.get('isSearchLimitReached')) {
this.loadHighlights();
}
},
Expand Down Expand Up @@ -67,6 +66,46 @@ polarity.export = PolarityComponent.extend({
},
toggleTabs: function (index) {
this.toggleProperty('details.results.' + index + '.showTabs', false);
},
setPage(fromIndex) {
this.runSearch(fromIndex);
}
},
updatePageParameters() {
let from = +this.get('details.from');
let size = +this.get('details.size');
let totalResults = this.get('details.totalResults');

if (totalResults <= size) {
this.set('block._state.paging.allResultsReturned', true);
}

this.set('block._state.paging.startItem', from + 1);
this.set('block._state.paging.endItem', from + size + 1 > totalResults ? totalResults : from + size);

const finalPagingIndex = totalResults % size === 0 ? totalResults - size : totalResults - (totalResults % size);
const nextPageIndex = from + size >= totalResults - 1 ? finalPagingIndex : from + size;
const prevPageIndex = from - size < 0 ? 0 : from - size;
const firstPageIndex = 0;
const lastPageIndex = size < totalResults ? finalPagingIndex : 0;

this.set('block._state.paging.nextPageIndex', nextPageIndex);
this.set('block._state.paging.prevPageIndex', prevPageIndex);
this.set('block._state.paging.firstPageIndex', firstPageIndex);
this.set('block._state.paging.lastPageIndex', lastPageIndex);

// There are no more pages to show so we can disable the next buttons
if (this.get('block._state.paging.endItem') === totalResults) {
this.set('block._state.paging.disableNextButtons', true);
} else {
this.set('block._state.paging.disableNextButtons', false);
}

// There are no more pages to show so we can disable the prev buttons
if (this.get('block._state.paging.startItem') === 1) {
this.set('block._state.paging.disablePrevButtons', true);
} else {
this.set('block._state.paging.disablePrevButtons', false);
}
},
_initSource(index) {
Expand Down Expand Up @@ -114,7 +153,8 @@ polarity.export = PolarityComponent.extend({
const payload = {
action: 'HIGHLIGHT',
documentIds,
entity: this.get('block.entity')
entity: this.get('block.entity'),
from: this.get('details.from')
};

this.sendIntegrationMessage(payload)
Expand All @@ -136,13 +176,16 @@ polarity.export = PolarityComponent.extend({
this.get('block').notifyPropertyChange('data');
});
},
runSearch() {
runSearch(from = 0) {
this.set('block._state.errorMessage', '');
this.set('block._state.searchRunning', true);
this.set('block._state.paging.disableNextButtons', true);
this.set('block._state.paging.disablePrevButtons', true);

const payload = {
action: 'SEARCH',
entity: this.get('block.entity')
entity: this.get('block.entity'),
from
};

this.sendIntegrationMessage(payload)
Expand All @@ -163,6 +206,7 @@ polarity.export = PolarityComponent.extend({
})
.finally(() => {
this.set('block._state.searchRunning', false);
this.updatePageParameters();
});
},
/**
Expand All @@ -175,11 +219,11 @@ polarity.export = PolarityComponent.extend({
},
initHighlights() {
this.get('details.results').forEach((result, index) => {
this._initSource(index);
Ember.set(result, 'showHighlights', false);
Ember.set(result, 'showTable', true);
Ember.set(result, 'showJson', false);
Ember.set(result, 'showSource', false);
this._initSource(index);
Ember.set(result, 'showHighlights', false);
Ember.set(result, 'showTable', true);
Ember.set(result, 'showJson', false);
Ember.set(result, 'showSource', false);
});
}
});
15 changes: 12 additions & 3 deletions config/config.js
View file
Expand Up @@ -110,13 +110,22 @@ module.exports = {
key: 'query',
name: 'Search Query',
description:
'The search query to execute as JSON. The top level property should be a `query` object and must be a valid JSON search request when sent to the ES `_search` REST endpoint.',
"The search query to execute as JSON. The top level property should be a `query` object and must be a valid JSON search request when sent to the ES `_search` REST endpoint. Use the 'Page Size' option to control 'size' and 'from' parameters.",
default:
'{"query": { "simple_query_string": { "query": "\\"{{entity}}\\"" } }, "from": 0, "size": 10, "sort": [ {"timestamp": "desc" } ] } }',
type: 'text',
userCanEdit: false,
adminOnly: true
},
{
key: 'defaultPageSize',
name: 'Page Size',
description: 'The number of results to display per page. This value must be between 1 and 100. Defaults to 10. This option should be set to "Only admins can view and edit".',
default: 10,
type: 'number',
userCanEdit: false,
adminOnly: true
},
{
key: 'highlightEnabled',
name: 'Enable Highlighting',
Expand All @@ -131,7 +140,7 @@ module.exports = {
key: 'highlightQuery',
name: 'Highlight Query',
description:
'The highlighter query to execute when a user clicks to view additional details. The top level property should be a `query` object. This query should typically match the query portion of your `Search Query`. Highlighting will attempt to highlight against all fields and will return the first 10 results. Only runs if the `Enable Highlighting` option is checked',
'The highlighter query to execute when a user clicks to view additional details. The top level property should be a `query` object. This query should typically match the query portion of your `Search Query`. Highlighting will attempt to highlight against all fields. Only runs if the `Enable Highlighting` option is checked',
default: '{"query": { "simple_query_string": { "query": "\\"{{entity}}\\"" } } }',
type: 'text',
userCanEdit: false,
Expand All @@ -151,7 +160,7 @@ module.exports = {
key: 'maxSummaryTags',
name: 'Maximum Number of Summary Tags',
description:
'The maximum number of summary tags to display in the Overlay Window before showing a count. If set to 0, all tags will be shown.',
'The maximum number of summary tags to display in the Overlay Window before showing a count. If set to 0, all tags will be shown.',
default: 5,
type: 'number',
userCanEdit: false,
Expand Down
13 changes: 11 additions & 2 deletions config/config.json
View file
Expand Up @@ -95,12 +95,21 @@
{
"key": "query",
"name": "Search Query",
"description": "The search query to execute as JSON. The top level property should be a `query` object and must be a valid JSON search request when sent to the ES `_search` REST endpoint.",
"description": "The search query to execute as JSON. The top level property should be a `query` object and must be a valid JSON search request when sent to the ES `_search` REST endpoint. Use the 'Page Size' option to control 'size' and 'from' parameters.",
"default": "{\"query\": { \"simple_query_string\": { \"query\": \"\\\"{{entity}}\\\"\" } }, \"from\": 0, \"size\": 10, \"sort\": [ {\"timestamp\": \"desc\" } ] } }",
"type": "text",
"userCanEdit": false,
"adminOnly": true
},
{
"key": "defaultPageSize",
"name": "Page Size",
"description": "The number of results to display per page. This value must be between 1 and 100. Defaults to 10. This option should be set to \"Only admins can view and edit\".",
"default": 10,
"type": "number",
"userCanEdit": false,
"adminOnly": true
},
{
"key": "highlightEnabled",
"name": "Enable Highlighting",
Expand All @@ -113,7 +122,7 @@
{
"key": "highlightQuery",
"name": "Highlight Query",
"description": "The highlighter query to execute when a user clicks to view additional details. The top level property should be a `query` object. This query should typically match the query portion of your `Search Query`. Highlighting will attempt to highlight against all fields and will return the first 10 results. Only runs if the `Enable Highlighting` option is checked",
"description": "The highlighter query to execute when a user clicks to view additional details. The top level property should be a `query` object. This query should typically match the query portion of your `Search Query`. Only runs if the `Enable Highlighting` option is checked",
"default": "{\"query\": { \"simple_query_string\": { \"query\": \"\\\"{{entity}}\\\"\" } } }",
"type": "text",
"userCanEdit": false,
Expand Down
52 changes: 42 additions & 10 deletions integration.js
View file
Expand Up @@ -86,6 +86,10 @@ function getAuthHeader(options, headers = {}) {
}
}

function parseErrorToReadableJSON(error) {
return JSON.parse(JSON.stringify(error, Object.getOwnPropertyNames(error)));
}

function doLookup(entities, options, cb) {
const filteredEntities = [];
let errors = [];
Expand Down Expand Up @@ -162,7 +166,7 @@ function doLookup(entities, options, cb) {
});
} else if (err) {
// a regular error occurred that is not a search limit related error
errors.push(err);
errors.push(parseErrorToReadableJSON(err));
// this error is returned for all the entities in the entity group so we need to count that entire group
// as having errors
errorCount += entityGroup.length;
Expand Down Expand Up @@ -198,6 +202,7 @@ function doLookup(entities, options, cb) {
errors
});
} else {
log.trace({ lookupResults }, 'Lookup Results');
cb(null, lookupResults);
}
}
Expand All @@ -207,7 +212,8 @@ function doLookup(entities, options, cb) {

function reachedSearchLimit(err, results) {
const maxRequestQueueLimitHit =
(_.isEmpty(err) && _.isEmpty(results)) || (err && err.message === 'This job has been dropped by Bottleneck');
((err === null || typeof err === 'undefined') && _.isEmpty(results)) ||
(err && err.message === 'This job has been dropped by Bottleneck');

let statusCode = Number.parseInt(_.get(err, 'errors.0.status', 0), 10);
const isGatewayTimeout = statusCode === 502 || statusCode === 504 || statusCode === 500;
Expand Down Expand Up @@ -279,12 +285,14 @@ function onMessage(payload, options, cb) {
switch (payload.action) {
case 'HIGHLIGHT':
if (options.highlightEnabled) {
options._fromIndex = payload.from;
loadHighlights(payload.entity, payload.documentIds, options, cb);
} else {
cb(null, {});
}
break;
case 'SEARCH':
options._fromIndex = payload.from;
doLookup([payload.entity], options, (searchErr, lookupResults) => {
if (searchErr) {
log.error({ searchErr }, 'Error running search');
Expand Down Expand Up @@ -344,10 +352,14 @@ function escapeEntityValue(entityValue) {
}

function _buildOnDetailsQuery(entityObj, documentIds, options) {
const highlightQuery = options.highlightQuery.replace(
entityTemplateReplacementRegex,
escapeEntityValue(entityObj.value)
const { queryString, from, size } = _getQueryWithPaging(
options.highlightQuery,
options.defaultPageSize,
options._fromIndex
);

const highlightQuery = queryString.replace(entityTemplateReplacementRegex, escapeEntityValue(entityObj.value));

return {
_source: false,
query: {
Expand All @@ -365,11 +377,20 @@ function _buildOnDetailsQuery(entityObj, documentIds, options) {
encoder: 'html',
fragment_size: 200
},
from: 0,
size: 10
from,
size
};
}

function _getQueryWithPaging(queryString, pageSize, fromIndex = 0) {
const queryObject = JSON.parse(queryString);

queryObject.from = fromIndex;
queryObject.size = pageSize;

return { queryString: JSON.stringify(queryObject), from: queryObject.from, size: queryObject.size };
}

/**
* Returns an elasticsearch query that uses the multi-search format:
*
Expand All @@ -383,13 +404,14 @@ function _buildOnDetailsQuery(entityObj, documentIds, options) {
function _buildDoLookupQuery(entities, options) {
let multiSearchString = '';
const multiSearchQueries = [];
const { queryString, from, size } = _getQueryWithPaging(options.query, options.defaultPageSize, options._fromIndex);

entities.forEach((entityObj) => {
const query = options.query.replace(entityTemplateReplacementRegex, escapeEntityValue(entityObj.value));
const query = queryString.replace(entityTemplateReplacementRegex, escapeEntityValue(entityObj.value));
multiSearchString += `{}\n${query}\n`;
multiSearchQueries.push(query);
});
return { multiSearchString, multiSearchQueries };
return { multiSearchString, multiSearchQueries, from, size };
}

function _getDetailBlockValues(hitResult) {
Expand Down Expand Up @@ -542,6 +564,9 @@ function _lookupEntityGroup(entityGroup, summaryFields, options, cb) {
data: {
summary: [],
details: {
totalResults: searchItemResult.hits.total.value,
from: queryObject.from,
size: queryObject.size,
results: hits,
tags: _getSummaryTags(searchItemResult, options),
queries: queryObject.multiSearchQueries
Expand Down Expand Up @@ -654,7 +679,7 @@ function _handleRestErrors(response, body) {
});
if (hasQueryError) {
return _createJsonErrorObject(
'Search query error encoutered. Please check your Search Query syntax.',
'Search query error encountered. Please check your Search Query syntax.',
null,
response.statusCode,
'7',
Expand Down Expand Up @@ -736,6 +761,13 @@ function validateOptions(userOptions, cb) {
}
}

if (userOptions.defaultPageSize.value < 1 || userOptions.defaultPageSize.value > 100) {
errors.push({
key: 'defaultPageSize',
message: 'The page size must be between 1 and 100.'
});
}

cb(null, errors);
}

Expand Down
2 changes: 1 addition & 1 deletion package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 61779ea

Please sign in to comment.