Skip to content

Commit

Permalink
fix: [performance] load analyst data in bulk
Browse files Browse the repository at this point in the history 
speeds up event loading dramatically
  • Loading branch information
@iglocska
iglocska committed Mar 18, 2024
1 parent 0723035 commit 0fb58cf
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 12 deletions.
43 changes: 41 additions & 2 deletions app/Model/Behavior/AnalystDataBehavior.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ class AnalystDataBehavior extends ModelBehavior

private $__current_type = null;

private $__valid_sharing_groups = null;

public function setup(Model $Model, $settings = array()) {
// We want to know whether we're a Note, Opinion or Relationship
$this->__current_type = $Model->alias;
Expand All @@ -22,15 +24,17 @@ public function fetchForUuid(Model $Model, $uuid, $user = null)
];
$type = $Model->current_type;
if (empty($user['Role']['perm_site_admin'])) {
$validSharingGroups = $Model->SharingGroup->authorizedIds($user, true);
if ($this->__valid_sharing_groups === null) {
$this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
}
$conditions['AND'][] = [
'OR' => [
$type . '.orgc_uuid' => $user['Organisation']['uuid'],
$type . '.org_uuid' => $user['Organisation']['uuid'],
$type . '.distribution IN' => [1, 2, 3],
'AND' => [
$type . '.distribution' => 4,
$type . '.sharing_group_id IN' => $validSharingGroups
$type . '.sharing_group_id IN' => $this->__valid_sharing_groups
]
]
];
Expand All @@ -42,6 +46,41 @@ public function fetchForUuid(Model $Model, $uuid, $user = null)
]);
}

// Return the analystData of the current type for a given UUID (this only checks the ACL of the analystData, NOT of the parent.)
public function fetchForUuids(Model $Model, $uuids, $user = null)
{
$conditions = [
'object_uuid' => $uuids
];
$type = $Model->current_type;
if (empty($user['Role']['perm_site_admin'])) {
if ($this->__valid_sharing_groups === null) {
$this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
}
$conditions['AND'][] = [
'OR' => [
$type . '.orgc_uuid' => $user['Organisation']['uuid'],
$type . '.org_uuid' => $user['Organisation']['uuid'],
$type . '.distribution IN' => [1, 2, 3],
'AND' => [
$type . '.distribution' => 4,
$type . '.sharing_group_id IN' => $this->__valid_sharing_groups
]
]
];
}
$temp = $Model->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'contain' => ['Org', 'Orgc', 'SharingGroup'],
]);
$results = [];
foreach ($temp as $result) {
$results[$result[$type]['object_uuid']] = $result;
}
return $results;
}

public function checkACL()
{

Expand Down
38 changes: 38 additions & 0 deletions app/Model/Behavior/AnalystDataParentBehavior.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,44 @@ public function attachAnalystData(Model $model, array $object, array $types = ['
return $data;
}

public function attachAnalystDataBulk(Model $model, array $objects, array $types = ['Note', 'Opinion', 'Relationship'])
{
$uuids = [];
$objects = array_chunk($objects, 10000);
if (empty($this->__currentUser)) {
$user_id = Configure::read('CurrentUserId');
$this->User = ClassRegistry::init('User');
if ($user_id) {
$this->__currentUser = $this->User->getAuthUser($user_id);
}
}
foreach ($objects as $chunk => $chunked_objects) {
foreach ($chunked_objects as $k => $object) {
if (!empty($object['uuid'])) {
$uuids[] = $object['uuid'];
}
}
// No uuids, nothing to attach
if (empty($uuids)) {
continue;
}
foreach ($types as $type) {
$this->{$type} = ClassRegistry::init($type);
$this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive);
$temp = $this->{$type}->fetchForUuids($uuids, $this->__currentUser);
if (!empty($temp)) {
foreach ($chunked_objects as $k => $object) {
if (!empty($temp[$object['uuid']])) {
$objects[$chunk][$k][$type][] = $temp[$object['uuid']][$type];
}
}
}
}
}
$objects = call_user_func_array('array_merge', $objects);
return $objects;
}

public function afterFind(Model $model, $results, $primary = false)
{
if (!empty($model->includeAnalystData)) {
Expand Down
14 changes: 4 additions & 10 deletions app/Model/Event.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2212,11 +2212,7 @@ public function fetchEvent($user, $options = array(), $useCache = false)
$event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
}
if (!empty($options['includeAnalystData'])) {
foreach ($event['Attribute'] as $k => $attribute) {
$this->Attribute->includeAnalystDataRecursive = true;
$analyst_data = $this->Attribute->attachAnalystData($attribute);
$event['Attribute'][$k] = array_merge($event['Attribute'][$k], $analyst_data);
}
$event['Attribute'] = $this->Attribute->attachAnalystDataBulk($event['Attribute']);
}

// move all object attributes to a temporary container
Expand Down Expand Up @@ -2269,11 +2265,9 @@ public function fetchEvent($user, $options = array(), $useCache = false)
if (isset($tempObjectAttributeContainer[$objectValue['id']])) {
$objectValue['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']];
}
if (!empty($options['includeAnalystData'])) {
$this->Object->includeAnalystDataRecursive = true;
$analyst_data = $this->Object->attachAnalystData($objectValue);
$objectValue = array_merge($objectValue, $analyst_data);
}
}
if (!empty($options['includeAnalystData'])) {
$event['Object'] = $this->Object->attachAnalystDataBulk($event['Object']);
}
unset($tempObjectAttributeContainer);
}
Expand Down

1 comment on commit 0fb58cf

@mdavis332
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @iglocska, fetchForUuids function block in app/Model/Behavior/AnalystDataBehavior.php may have been indented one too many times. i don't think it causes functional issues but does disrupt flow of reading.

Please sign in to comment.