Skip to content

Commit

Permalink
Merge pull request #8415 from JakubOnderka/faster-search
Browse files Browse the repository at this point in the history 
Faster search
  • Loading branch information
@JakubOnderka
JakubOnderka committed May 30, 2022
2 parents 21b4348 + 9675c93 commit d68d3d4
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 62 deletions.
115 changes: 58 additions & 57 deletions app/Controller/EventsController.php
View file
Expand Up @@ -1214,45 +1214,6 @@ public function filterEventIndex()
$this->layout = false;
}

/**
* Search for a value on an attribute level for a specific field.
*
* @param array $attribute An attribute
* @param array $fields List of keys in attribute to search in
* @param array $searchParts Values to search
* @return bool Returns true on match
*/
private function __valueInFieldAttribute($attribute, $fields, $searchParts)
{
foreach ($fields as $field) {
if (strpos($field, 'Tag') === 0) {
if (empty($attribute['AttributeTag'])) {
continue;
}
$fieldValues = Hash::extract($attribute, 'AttributeTag.{n}.' . $field);
foreach ($fieldValues as $fieldValue) {
$fieldValue = mb_strtolower($fieldValue);
foreach ($searchParts as $s) {
if (strpos($fieldValue, $s) !== false) {
return true;
}
}
}
} else {
if (!isset($attribute[$field])) {
continue;
}
$fieldValue = mb_strtolower($attribute[$field]);
foreach ($searchParts as $s) {
if (strpos($fieldValue, $s) !== false) {
return true;
}
}
}
}
return false;
}

public function viewEventAttributes($id, $all = false)
{
$filterData = array(
Expand Down Expand Up @@ -2004,15 +1965,15 @@ public function removePivot($id, $eventId, $self = false)
* @param string $searchFor
* @param string|false $filterColumnsOverwrite
*/
private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite=false)
private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite = false)
{
// filtering on specific columns is specified
if ($filterColumnsOverwrite !== false) {
$filterValue = array_map('trim', explode(",", $filterColumnsOverwrite));
} else {
$filterColumnsOverwrite = Configure::read('MISP.event_view_filter_fields') ?: 'id,uuid,value,comment,type,category,Tag.name';
$filterValue = array_map('trim', explode(",", $filterColumnsOverwrite));
$validFilters = array('id', 'uuid', 'value', 'comment', 'type', 'category', 'Tag.name');
$validFilters = ['id', 'uuid', 'value', 'comment', 'type', 'category', 'Tag.name'];
foreach ($filterValue as $k => $v) {
if (!in_array($v, $validFilters, true)) {
unset($filterValue[$k]);
Expand All @@ -2023,41 +1984,81 @@ private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite
$searchParts = explode('|', mb_strtolower($searchFor));

// search in all attributes
foreach ($event['Attribute'] as $k => $attribute) {
if (!$this->__valueInFieldAttribute($attribute, $filterValue, $searchParts)) {
unset($event['Attribute'][$k]);
$foundAttributes = [];
foreach ($event['Attribute'] as $attribute) {
if ($this->__valueInFieldAttribute($attribute, $filterValue, $searchParts)) {
$foundAttributes[] = $attribute;
}
}
$event['Attribute'] = array_values($event['Attribute']);
$event['Attribute'] = $foundAttributes;

// search in all attributes
foreach ($event['ShadowAttribute'] as $k => $proposals) {
if (!$this->__valueInFieldAttribute($proposals, $filterValue, $searchParts)) {
unset($event['ShadowAttribute'][$k]);
// search in all proposals
$foundProposals = [];
foreach ($event['ShadowAttribute'] as $proposals) {
if ($this->__valueInFieldAttribute($proposals, $filterValue, $searchParts)) {
$foundProposals[] = $proposals;
}
}
$event['ShadowAttribute'] = array_values($event['ShadowAttribute']);
$event['ShadowAttribute'] = $foundProposals;

// search for all attributes in object
foreach ($event['Object'] as $k => $object) {
if ($this->__valueInFieldAttribute($object, ['id', 'uuid', 'name', 'comment'], $searchParts)) {
continue;
}
foreach ($object['Attribute'] as $k2 => $attribute) {
if (!$this->__valueInFieldAttribute($attribute, $filterValue, $searchParts)) {
unset($event['Object'][$k]['Attribute'][$k2]);
$foundAttributes = [];
foreach ($object['Attribute'] as $attribute) {
if ($this->__valueInFieldAttribute($attribute, $filterValue, $searchParts)) {
$foundAttributes[] = $attribute;
}
}
if (empty($event['Object'][$k]['Attribute'])) {
// remove object if empty
unset($event['Object'][$k]);
if (empty($foundAttributes)) {
unset($event['Object'][$k]); // remove object if contains no attributes
} else {
$event['Object'][$k]['Attribute'] = array_values($event['Object'][$k]['Attribute']);
$event['Object'][$k]['Attribute'] = $foundAttributes;
}
}
$event['Object'] = array_values($event['Object']);
}

/**
* Search for a value on an attribute level for a specific field.
*
* @param array $attribute An attribute
* @param array $fields List of keys in attribute to search in
* @param array $searchParts Values to search (OR)
* @return bool Returns true on match
*/
private function __valueInFieldAttribute($attribute, $fields, $searchParts)
{
foreach ($fields as $field) {
if ($field === 'Tag.name') {
if (empty($attribute['AttributeTag'])) {
continue;
}
foreach ($attribute['AttributeTag'] as $fieldValue) {
$fieldValue = mb_strtolower($fieldValue['Tag']['name']);
foreach ($searchParts as $s) {
if (strpos($fieldValue, $s) !== false) {
return true;
}
}
}
} else {
if (!isset($attribute[$field])) {
continue;
}
$fieldValue = mb_strtolower($attribute[$field]);
foreach ($searchParts as $s) {
if (strpos($fieldValue, $s) !== false) {
return true;
}
}
}
}
return false;
}

// look in the parameters if we are doing advanced filtering or not
private function __checkIfAdvancedFiltering($filters)
{
Expand Down
10 changes: 5 additions & 5 deletions app/Model/Event.php
View file
Expand Up @@ -1486,7 +1486,7 @@ public function filterEventIds($user, &$params = array(), &$result_count = 0)
);
foreach ($params as $param => $paramData) {
foreach ($simple_params as $scope => $simple_param_scoped) {
if (isset($simple_param_scoped[$param]) && $params[$param] !== false) {
if (isset($simple_param_scoped[$param]) && $paramData !== false) {
$options = array(
'filter' => $param,
'scope' => $scope,
Expand Down Expand Up @@ -2828,14 +2828,14 @@ public function set_filter_attribute_id(&$params, $conditions, $options)
return $conditions;
}

public function set_filter_value(&$params, $conditions, $options, $keys = array('Attribute.value1', 'Attribute.value2'))
public function set_filter_value(&$params, $conditions, $options)
{
if (!empty($params['value'])) {
$valueParts = explode('|', $params['value'], 2);
$params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], $keys);
$conditions = $this->generic_add_filter($conditions, $params[$options['filter']], ['Attribute.value1', 'Attribute.value2']);
// Allows searching for ['value1' => [full, part1], 'value2' => [full, part2]]
if (count($valueParts) == 2) {
if (is_string($params['value']) && strpos('|', $params['value']) !== false) {
$valueParts = explode('|', $params['value'], 2);
$convertedFilterVal1 = $this->convert_filters($valueParts[0]);
$convertedFilterVal2 = $this->convert_filters($valueParts[1]);
$conditionVal1 = $this->generic_add_filter([], $convertedFilterVal1, ['Attribute.value1'])['AND'][0]['OR'];
Expand Down

0 comments on commit d68d3d4

Please sign in to comment.