546 changes: 311 additions & 235 deletions INSTALL/INSTALL.sh
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions INSTALL/INSTALL.sh.sfv
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
; Generated by RHash v1.3.8 on 2020-04-30 at 15:20.13
; Generated by RHash v1.3.9 on 2020-05-17 at 03:12.57
; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
;
; 131010 15:20.13 2020-04-30 INSTALL.sh
INSTALL.sh 660E0D51D88B57CE5BE725117482207E39371038 DCF69118CD37B43C308FD25E6BADAF03549BAF0FFA2AC11A1E919005D700F4AC 74E03A8054AF2E4BCB90426A3B813F57BF032734AB7B4E9D4F6F96961D7371FB051180BEE8357642EB9CC58603C13DA3 C4D1D02980808A92E8E11C72A49AA354DDEFA71C6E85FAC739645CEDEB4B36415243F7FB4B8BC75B6AE7B5D9660E0F88A35E884EBD51EA107128B0D7FB20C946
; 132940 03:12.57 2020-05-17 INSTALL.sh
INSTALL.sh 89D2EC44902AE611FB65EC5CF30FD8685F713ECD 4D2C48FBF6D668A0FF35B01DE88BAA656943C13A1CEB148C09DD333299E894B3 9BC9C39F584346C8EC1DBF87DBF0E1952EF1736DF22E6BE233A60968F751E2C1C5A8E36CE34BE5696D6EE61D8E1079FA 0AD43002838AF3EB559FC2493A9F9B9FFC1E9592311E0866A46ADA3FB3D0D2AF419B06EB42454184E4DDEDDFAC281C07148E4C61277F933D942ABCEA7CF976CA
2 changes: 1 addition & 1 deletion INSTALL/INSTALL.sh.sha1
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
660e0d51d88b57ce5be725117482207e39371038 INSTALL.sh
89d2ec44902ae611fb65ec5cf30fd8685f713ecd INSTALL.sh
2 changes: 1 addition & 1 deletion INSTALL/INSTALL.sh.sha256
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
dcf69118cd37b43c308fd25e6badaf03549baf0ffa2ac11a1e919005d700f4ac INSTALL.sh
4d2c48fbf6d668a0ff35b01de88baa656943c13a1ceb148c09dd333299e894b3 INSTALL.sh
2 changes: 1 addition & 1 deletion INSTALL/INSTALL.sh.sha384
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
74e03a8054af2e4bcb90426a3b813f57bf032734ab7b4e9d4f6f96961d7371fb051180bee8357642eb9cc58603c13da3 INSTALL.sh
9bc9c39f584346c8ec1dbf87dbf0e1952ef1736df22e6be233a60968f751e2c1c5a8e36ce34be5696d6ee61d8e1079fa INSTALL.sh
2 changes: 1 addition & 1 deletion INSTALL/INSTALL.sh.sha512
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c4d1d02980808a92e8e11c72a49aa354ddefa71c6e85fac739645cedeb4b36415243f7fb4b8bc75b6ae7b5d9660e0f88a35e884ebd51ea107128b0d7fb20c946 INSTALL.sh
0ad43002838af3eb559fc2493a9f9b9ffc1e9592311e0866a46ada3fb3d0d2af419b06eb42454184e4ddeddfac281c07148e4c61277f933d942abcea7cf976ca INSTALL.sh
242 changes: 116 additions & 126 deletions INSTALL/INSTALL.tpl.sh
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion PyMISP
Submodule PyMISP updated 14 files
+56 −0 CHANGELOG.txt
+87 −71 poetry.lock
+3 −3 pymisp/__init__.py
+34 −20 pymisp/abstract.py
+408 −303 pymisp/api.py
+1 −1 pymisp/data/misp-objects
+60 −39 pymisp/mispevent.py
+3 −0 pymisp/tools/__init__.py
+1 −1 pymisp/tools/csvloader.py
+144 −0 pymisp/tools/microblogobject.py
+3 −3 pymisp/tools/sshauthkeyobject.py
+3 −2 pyproject.toml
+189 −6 tests/testlive_comprehensive.py
+1 −1 travis/install_travis.sh
2 changes: 1 addition & 1 deletion VERSION.json
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"major":2, "minor":4, "hotfix":125}
{"major":2, "minor":4, "hotfix":126}
12 changes: 2 additions & 10 deletions app/Console/Command/AuthkeyShell.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,8 @@ public function main()
$this->User->id = $user['User']['id'];
$newkey = $this->User->generateAuthKey();
if ($this->User->saveField('authkey', $newkey)) {
$this->Log->create();
$this->Log->save(array(
'org' => $user['Organisation']['name'],
'model' => 'User',
'model_id' => $user['User']['id'],
'email' => 'SYSTEM',
'action' => 'reset_auth_key',
'title' => 'Authentication key for user ' . $user['User']['id'] . ' (' . $user['User']['email'] . ')',
'change' => 'authkey(' . $user['User']['authkey'] . ') => (' . $newkey . ')'
));
$logTitle = 'Authentication key for user ' . $user['User']['id'] . ' (' . $user['User']['email'] . ')';
$this->Log->createLogEntry('SYSTEM', 'reset_auth_key', 'User', $user['User']['id'], $logTitle, array('authkey' => array($user['User']['authkey'], $newkey)));
echo $newkey . PHP_EOL;
} else {
echo 'Could not update account for User.id = ', $user['User']['id'], PHP_EOL;
Expand Down
6 changes: 5 additions & 1 deletion app/Console/Command/ServerShell.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,13 @@ public function pull()
$this->Job->save($data);
$jobId = $this->Job->id;
}
$force = false;
if (!empty($this->args[4]) && $this->args[4] === 'force') {
$force = true;
}
$this->Server->id = $serverId;
$server = $this->Server->read(null, $serverId);
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId);
$result = $this->Server->pull($user, $serverId, $technique, $server, $jobId, $force);
$this->Job->id = $jobId;
$this->Job->save(array(
'id' => $jobId,
Expand Down
4 changes: 2 additions & 2 deletions app/Controller/AppController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,8 @@ class AppController extends Controller

public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');

private $__queryVersion = '105';
public $pyMispVersion = '2.4.125';
private $__queryVersion = '106';
public $pyMispVersion = '2.4.126';
public $phpmin = '7.2';
public $phprec = '7.4';
public $pythonmin = '3.6';
Expand Down
6 changes: 3 additions & 3 deletions app/Controller/AttributesController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ public function add($eventId = false)
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
if ($this->request->data['Attribute']['distribution'] == 4) {
if (isset($this->request->data['Attribute']['distribution']) && $this->request->data['Attribute']['distribution'] == 4) {
$sg = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']);
if (empty($sg)) {
throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.'));
Expand Down Expand Up @@ -837,7 +837,7 @@ public function edit($id = null)
if (!isset($this->request->data['Attribute'])) {
$this->request->data = array('Attribute' => $this->request->data);
}
if ($this->request->data['Attribute']['distribution'] == 4) {
if (isset($this->request->data['Attribute']['distribution']) && $this->request->data['Attribute']['distribution'] == 4) {
$sg = $this->Attribute->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1, $this->request->data['Attribute']['sharing_group_id']);
if (empty($sg)) {
throw new MethodNotAllowedException(__('Invalid Sharing Group or not authorised.'));
Expand Down Expand Up @@ -3148,7 +3148,7 @@ public function removeTag($id = false, $tag_id = false)

public function toggleCorrelation($id)
{
if (!$this->_isSiteAdmin() && Configure::read('MISP.allow_disabling_correlation')) {
if (!$this->_isSiteAdmin() && !Configure::read('MISP.allow_disabling_correlation')) {
throw new MethodNotAllowedException(__('Disabling the correlation is not permitted on this instance.'));
}
$this->Attribute->id = $id;
Expand Down
31 changes: 16 additions & 15 deletions app/Controller/Component/ACLComponent.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,21 +285,22 @@ class ACLComponent extends Component
'view' => array('*')
),
'objects' => array(
'add' => array('perm_add'),
'addValueField' => array('perm_add'),
'delete' => array('perm_add'),
'edit' => array('perm_add'),
'get_row' => array('perm_add'),
'orphanedObjectDiagnostics' => array(),
'editField' => array('perm_add'),
'fetchEditForm' => array('perm_add'),
'fetchViewValue' => array('*'),
'quickAddAttributeForm' => array('perm_add'),
'quickFetchTemplateWithValidObjectAttributes' => array('perm_add'),
'proposeObjectsFromAttributes' => array('*'),
'groupAttributesIntoObject' => array('perm_add'),
'revise_object' => array('perm_add'),
'view' => array('*'),
'add' => array('perm_add'),
'addValueField' => array('perm_add'),
'delete' => array('perm_add'),
'edit' => array('perm_add'),
'get_row' => array('perm_add'),
'orphanedObjectDiagnostics' => array(),
'editField' => array('perm_add'),
'fetchEditForm' => array('perm_add'),
'fetchViewValue' => array('*'),
'quickAddAttributeForm' => array('perm_add'),
'quickFetchTemplateWithValidObjectAttributes' => array('perm_add'),
'restSearch' => array('*'),
'proposeObjectsFromAttributes' => array('*'),
'groupAttributesIntoObject' => array('perm_add'),
'revise_object' => array('perm_add'),
'view' => array('*'),
),
'objectReferences' => array(
'add' => array('perm_add'),
Expand Down
8 changes: 4 additions & 4 deletions app/Controller/Component/RestResponseComponent.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class RestResponseComponent extends Component
'restSearch' => array(
'description' => "Search MISP using a list of filter parameters and return the data in the selected format. The search is available on an event and an attribute level, just select the scope via the URL (/events/restSearch vs /attributes/restSearch). Besides the parameters listed, other, format specific ones can be passed along (for example: requested_attributes and includeContext for the CSV export). This API allows pagination via the page and limit parameters.",
'mandatory' => array('returnFormat'),
'optional' => array('page', 'limit', 'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'date', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'eventinfo', 'excludeLocalTags'),
'optional' => array('page', 'limit', 'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'date', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'eventinfo', 'excludeLocalTags', 'threat_level_id'),
'params' => array()
)
),
Expand Down Expand Up @@ -1080,7 +1080,7 @@ private function __configureFieldConstraints()
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False' ),
'help' => __('Will not return Attributes, shadow attribute and objects')
'help' => __('Will only return the metadata of the given query scope, contained data is omitted.')
),
'minimal' => array(
'input' => 'radio',
Expand Down Expand Up @@ -1456,13 +1456,13 @@ private function __configureFieldConstraints()
'input' => 'select',
'type' => 'integer',
'operators' => ['equal', 'not_equal'],
'values' => array( 1 => 'Hight', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
'values' => array( 1 => 'High', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
),
'threatlevel' => array(
'input' => 'select',
'type' => 'integer',
'operators' => ['equal', 'not_equal'],
'values' => array( 1 => 'Hight', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
'values' => array( 1 => 'High', 2 => 'Medium', 3 => 'Low', 4 => 'Undefined')
),
'time' => array(
'input' => 'text',
Expand Down
5 changes: 3 additions & 2 deletions app/Controller/Component/RestSearchComponent.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@ class RestSearchComponent extends Component
'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen'
'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen',
'threat_level_id'
),
'Event' => array(
'returnFormat', 'value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
'metadata', 'uuid', 'publish_timestamp', 'timestamp', 'published', 'enforceWarninglist', 'sgReferenceOnly',
'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'to_ids', 'deleted',
'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation'
'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation', 'threat_level_id'
),
'Object' => array(
'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
Expand Down
2 changes: 1 addition & 1 deletion app/Controller/EventsController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5244,7 +5244,7 @@ public function exportModule($module, $id, $standard = false)

public function toggleCorrelation($id)
{
if (!$this->_isSiteAdmin() && Configure.read('MISP.allow_disabling_correlation')) {
if (!$this->_isSiteAdmin() && !Configure::read('MISP.allow_disabling_correlation')) {
throw new MethodNotAllowedException(__('Disabling the correlation is not permitted on this instance.'));
}
$this->Event->id = $id;
Expand Down
6 changes: 5 additions & 1 deletion app/Controller/FeedsController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,7 +364,11 @@ public function edit($feedId)
}
}
if (!isset($this->request->data['Feed']['settings'])) {
$this->request->data['Feed']['settings'] = array();
if (!empty($this->Feed->data['Feed']['settings'])) {
$this->request->data['Feed']['settings'] = $this->Feed->data['Feed']['settings'];
} else {
$this->request->data['Feed']['settings'] = array();
}
} else {
if (!empty($this->request->data['Feed']['settings']['common']['excluderegex']) && !$this->__checkRegex($this->request->data['Feed']['settings']['common']['excluderegex'])) {
$this->Flash->error('Invalid exclude regex. Make sure it\'s a delimited PCRE regex pattern.');
Expand Down
1 change: 1 addition & 0 deletions app/Controller/ServersController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1624,6 +1624,7 @@ public function statusZeroMQServer()
$result = $pubSubTool->statusCheck();
if (!empty($result)) {
$this->set('events', $result['publishCount']);
$this->set('messages', $result['messageCount']);
$this->set('time', date('Y/m/d H:i:s', $result['timestamp']));
$this->set('time2', date('Y/m/d H:i:s', $result['timestampSettings']));
}
Expand Down
21 changes: 15 additions & 6 deletions app/Controller/UsersController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1840,6 +1840,7 @@ private function __statisticsData($params = array())
$stats['user_count_pgp'] = $this->User->find('count', array('recursive' => -1, 'conditions' => array('User.gpgkey !=' => '')));
$stats['org_count'] = count($orgs);
$stats['local_org_count'] = count($local_orgs);
$stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
$stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);

$this->loadModel('Thread');
Expand Down Expand Up @@ -2373,7 +2374,7 @@ public function register()
$requestObject['message'] = '';
}
if (empty($requestObject['email'])) {
throw new InvalidArgumentException(__('We require at least the email field to be filled.'));
throw new BadRequestException(__('We require at least the email field to be filled.'));
}
$this->loadModel('Inbox');
$this->Inbox->create();
Expand Down Expand Up @@ -2472,8 +2473,13 @@ public function discardRegistrations($id = false)
$id = $this->params['named']['id'];
}
$this->loadModel('Inbox');
if (Validation::uuid($id)) {
$id = $this->Toolbox->findIdByUuid($this->Inbox, $id);
if (!is_array($id)) {
$id = array($id);
}
foreach ($id as $k => $v) {
if (Validation::uuid($v)) {
$id[$k] = $this->Toolbox->findIdByUuid($this->Inbox, $v);
}
}
$registrations = $this->Inbox->find('all', array(
'recursive' => -1,
Expand All @@ -2498,7 +2504,7 @@ public function discardRegistrations($id = false)
$this->Log->save(array(
'org' => $this->Auth->user('Organisation')['name'],
'model' => 'User',
'model_id' => $id,
'model_id' => 0,
'email' => $this->Auth->user('email'),
'action' => 'discardRegistrations',
'title' => $message,
Expand Down Expand Up @@ -2564,6 +2570,9 @@ public function acceptRegistrations($id = false)
'perm_admin' => $role['Role']['perm_admin']
);
}
if (empty($this->request->data['User'])) {
$this->request->data = array('User' => $this->request->data);
}
if (!empty($default_role)) {
$this->request->data['User']['role_id'] = $default_role['Role']['id'];
}
Expand All @@ -2590,11 +2599,11 @@ public function acceptRegistrations($id = false)
if (!empty($default_role)) {
$this->request->data['User']['role_id'] = $default_role['Role']['id'];
} else {
throw new InvalidArgumentException(__('Role ID not provided and no default role exist on the instance'));
throw new BadRequestException(__('Role ID not provided and no default role exist on the instance'));
}
}
if (!isset($this->request->data['User']['org_id'])) {
throw new InvalidArgumentException(__('No organisation selected. Supply an Organisation ID'));
throw new BadRequestException(__('No organisation selected. Supply an Organisation ID'));
} else {
if (Validation::uuid($this->request->data['User']['org_id'])) {
$id = $this->Toolbox->findIdByUuid($this->User->Organisation, $this->request->data['User']['org_id']);
Expand Down
2 changes: 1 addition & 1 deletion app/Lib/Export/JsonExport.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public function handler($data, $options = array())
} else if($options['scope'] === 'Event') {
return $this->__eventHandler($data, $options);
} else if($options['scope'] === 'Object') {
return $this->__eventHandler($data, $options);
return $this->__objectHandler($data, $options);
} else if($options['scope'] === 'Sighting') {
return $this->__sightingsHandler($data, $options);
}
Expand Down
130 changes: 130 additions & 0 deletions app/Lib/Export/OpendataExport.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
<?php

class OpendataExport
{
public $non_restrictive_export = true;
public $use_default_filters = true;
public $mock_query_only = true;
private $__default_filters = null;

private $__auth = null;
private $__delete = false;
private $__scope = null;
private $__setup = array();
private $__url = null;

private $__scripts_dir = APP . 'files/scripts/';
private $__script_name = 'misp-opendata/opendata.py';

public function setDefaultFilters($filters)
{
$this->__default_filters = $filters;
}

public function header($options = array())
{
$this->__scope = $options['scope'];
if (isset($this->__default_filters['auth'])) {
$this->__auth = $this->__default_filters['auth'];
unset($this->__default_filters['auth']);
}
if (isset($this->__default_filters['setup'])) {
$this->__setup = $this->__default_filters['setup'];
$this->__check_setup_filter();
unset($this->__default_filters['setup']);
} else {
throw new Exception(__('Missing "setup" filter containing the dataset and resource(s) information.'));
}
if (isset($this->__default_filters['url'])) {
$this->__url = $this->__default_filters['url'];
unset($this->__default_filters['url']);
} else {
$external_baseurl = Configure::read('MISP.external_baseurl');
$baseurl = !empty($external_baseurl) ? $external_baseurl : Configure::read('MISP.baseurl');
if (empty($baseurl)) {
throw new Exception(__('Missing url of the MISP instance, and baseurl is not set.'));
}
$this->__url = $baseurl;
}
if (!empty($this->__default_filters['delete'])) {
$this->__delete = true;
unset($this->__default_filters['delete']);
}
return '';
}

public function footer()
{
$authParam = ' --auth ' . $this->__auth;
$my_server = ClassRegistry::init('Server');
$cmd = $my_server->getPythonVersion() . ' ' . $this->__scripts_dir . $this->__script_name . $authParam;
return $this->__delete ? $this->__delete_query($cmd) : $this->__add_query($cmd);
}

public function separator()
{
return '';
}

private function __add_query($cmd)
{
unset($this->__default_filters['returnFormat']);
$body = json_encode($this->__default_filters);
$bodyFilename = $this->__generateSetupFile($body);
$bodyParam = ' --body ' . $bodyFilename;
$levelParam = ' --level ' . strtolower($this->__scope) . 's';
$setup = json_encode($this->__setup);
$setupFilename = $this->__generateSetupFile($setup);
$setupParam = ' --setup ' . $setupFilename;
$urlParam = ' --url ' . $this->__url;

$cmd .= $bodyParam . $setupParam . $levelParam . $urlParam;
$results = shell_exec($cmd);
unlink($bodyFilename);
unlink($setupFilename);
return $results;
}

private function __check_setup_filter()
{
if (empty($this->__setup['dataset'])) {
throw new Exception(__('Missing dataset filter in the setup filter. Please provide the dataset setup.'));
}
if (!empty($this->__setup['resources']) && !empty($this->__setup['resource'])) {
throw new Exception(__('Please provide the resource setup in a single field called "resources".'));
}
if (!empty($this->__setup['resource']) && empty($this->__setup['resources'])) {
$this->__setup['resources'] = $this->__setup['resource'];
unset($this->__setup['resource']);
}
}

private function __delete_query($cmd)
{
$cmd .= " -d '" . $this->__setup['dataset'] . "'";
if (!empty($this->__setup['resources'])) {
if (is_array($this->__setup['resources'])) {
foreach ($this->__setup['resources'] as $resource) {
$cmd .= ' ' . $resource;
}
} else {
$cmd .= " '" . $this->__setup['resources'] . "'";
}
}
return shell_exec($cmd);
}

private function __generateRandomFileName()
{
return (new RandomTool())->random_str(false, 12);
}

private function __generateSetupFile($to_write)
{
$filename = $this->__scripts_dir . 'tmp/' . $this->__generateRandomFileName();
$tmpFile = new File($filename, true, 0644);
$tmpFile->write($to_write);
$tmpFile->close();
return $filename;
}
}
10 changes: 6 additions & 4 deletions app/Lib/Tools/JSONConverterTool.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,12 @@ public function convert($event, $isSiteAdmin=false, $raw = false)
unset($event['Event']['SharingGroup']);
}
if ($object == 'Galaxy') {
foreach ($event['Event']['Galaxy'] as $k => $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $k2 => $cluster) {
if (empty($cluster['meta'])) {
$event['Event']['Galaxy'][$k]['GalaxyCluster'][$k2]['meta'] = new stdclass();
if (!empty($event['Event']['Galaxy'])) {
foreach ($event['Event']['Galaxy'] as $k => $galaxy) {
foreach ($galaxy['GalaxyCluster'] as $k2 => $cluster) {
if (empty($cluster['meta'])) {
$event['Event']['Galaxy'][$k]['GalaxyCluster'][$k2]['meta'] = new stdclass();
}
}
}
}
Expand Down
263 changes: 153 additions & 110 deletions app/Lib/Tools/PubSubTool.php
View file
Original file line number Diff line number Diff line change
@@ -1,85 +1,68 @@
<?php
class PubSubTool
{
private $__redis = false;
private $__settings = false;
const SCRIPTS_TMP = APP . 'files' . DS . 'scripts' . DS . 'tmp' . DS;
const OLD_PID_LOCATION = APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.pid';

private function __getSetSettings()
{
$settings = array(
'redis_host' => 'localhost',
'redis_port' => '6379',
'redis_password' => '',
'redis_database' => '1',
'redis_namespace' => 'mispq',
'port' => '50000',
);
/**
* @var Redis
*/
private $redis;

foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
if ($temp) {
$settings[$key] = $temp;
}
}
$settingsFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'settings.json', true, 0644);
$settingsFile->write(json_encode($settings, true));
$settingsFile->close();
return $settings;
}
/**
* @var array
*/
private $settings;

public function initTool()
{
if (!$this->__redis) {
$settings = $this->__setupPubServer();
$redis = new Redis();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$this->__redis = $redis;
$this->__settings = $settings;
} else {
$settings = $this->__settings;
if (!$this->redis) {
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
$this->redis = $this->createRedisConnection($settings);
$this->settings = $settings;
}
return $settings;
}

// read the pid file, if it exists, check if the process is actually running
// if either the pid file doesn't exists or the process is not running return false
// otherwise return the pid
public function checkIfRunning()
/**
* Read the pid file, if it exists, check if the process is actually running
* if either the pid file doesn't exists or the process is not running return false
* otherwise return the pid.
*
* @param string|null $pidFilePath
* @return bool|int False when process is not running, PID otherwise.
* @throws Exception
*/
public function checkIfRunning($pidFilePath = null)
{
$pidFile = new File(APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.pid');
$pid = $pidFile->read(true, 'r');
$pidFile = new File($pidFilePath ?: self::SCRIPTS_TMP . 'mispzmq.pid');
if (!$pidFile->exists()) {
return false;
}
$pid = $pidFile->read();
if ($pid === false || $pid === '') {
return false;
}
if (!is_numeric($pid)) {
throw new Exception('Internal error (invalid PID file for the MISP zmq script)');
}
$result = trim(shell_exec('ps aux | awk \'{print $2}\' | grep "^' . $pid . '$"'));
if (empty($result)) {
$result = file_exists("/proc/$pid");
if ($result === false) {
return false;
}
return $pid;
}

public function statusCheck()
{
$redis = new Redis();
$settings = $this->__getSetSettings();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'status');
sleep(1);
$response = trim($redis->lPop($settings['redis_namespace'] . ':status'));
return json_decode($response, true);
$response = $redis->blPop($settings['redis_namespace'] . ':status', 5);
if ($response === null) {
throw new Exception("No response from status command returned after 5 seconds.");
}
return json_decode(trim($response[1]), true);
}

public function checkIfPythonLibInstalled()
Expand All @@ -92,139 +75,113 @@ public function checkIfPythonLibInstalled()
return false;
}

private function __setupPubServer()
{
App::uses('File', 'Utility');
$my_server = ClassRegistry::init('Server');
$settings = $this->__getSetSettings();
if ($this->checkIfRunning() === false) {
shell_exec($my_server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py > ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
}
return $settings;
}

public function publishEvent($event)
{
App::uses('JSONConverterTool', 'Tools');
$jsonTool = new JSONConverterTool();
$json = $jsonTool->convert($event);
return $this->__pushToRedis(':data:misp_json', $json);
return $this->pushToRedis(':data:misp_json', $json);
}

public function event_save($event, $action)
{
if (!empty($action)) {
$event['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_event', json_encode($event, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_event', json_encode($event, JSON_PRETTY_PRINT));
}

public function object_save($object, $action)
{
if (!empty($action)) {
$object['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_object', json_encode($object, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_object', json_encode($object, JSON_PRETTY_PRINT));
}

public function object_reference_save($object_reference, $action)
{
if (!empty($action)) {
$object_reference['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_object_reference', json_encode($object_reference, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_object_reference', json_encode($object_reference, JSON_PRETTY_PRINT));
}

public function publishConversation($message)
{
return $this->__pushToRedis(':data:misp_json_conversation', json_encode($message, JSON_PRETTY_PRINT));
}

private function __pushToRedis($ns, $data)
{
$this->__redis->rPush($this->__settings['redis_namespace'] . $ns, $data);
return true;
return $this->pushToRedis(':data:misp_json_conversation', json_encode($message, JSON_PRETTY_PRINT));
}

public function attribute_save($attribute, $action = false)
{
if (!empty($action)) {
$attribute['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_attribute', json_encode($attribute, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_attribute', json_encode($attribute, JSON_PRETTY_PRINT));
}

public function tag_save($tag, $action = false)
{
if (!empty($action)) {
$tag['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_tag', json_encode($tag, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_tag', json_encode($tag, JSON_PRETTY_PRINT));
}

public function sighting_save($sighting, $action = false)
{
if (!empty($action)) {
$sighting['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_sighting', json_encode($sighting, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_sighting', json_encode($sighting, JSON_PRETTY_PRINT));
}

public function modified($data, $type, $action = false)
{
if (!empty($action)) {
$data['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
}

public function publish($data, $type, $action = false)
{
if (!empty($action)) {
$data['action'] = $action;
}
return $this->__pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
return $this->pushToRedis(':data:misp_json_' . $type, json_encode($data, JSON_PRETTY_PRINT));
}

public function killService($settings = false)
public function killService()
{
$redis = new Redis();
if ($this->checkIfRunning()) {
if ($settings == false) {
$settings = $this->__getSetSettings();
}
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$settings = $this->getSetSettings();
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
sleep(1);
if ($this->checkIfRunning()) {
// Still running.
return false;
}
}
return true;
}

// reload the server if it is running, if not, start it
/**
* Reload the server if it is running, if not, start it.
*
* @return bool|string
* @throws Exception
*/
public function reloadServer()
{
if (!$this->checkIfRunning()) {
$settings = $this->__setupPubServer();
} else {
$settings = $this->__getSetSettings();
$redis = new Redis();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redis_pwd = $settings['redis_password'];
if (!empty($redis_pwd)) {
$redis->auth($redis_pwd);
}
$redis->select($settings['redis_database']);
$settings = $this->getSetSettings();
$this->saveSettingToFile($settings);

if ($this->checkIfRunning()) {
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'reload');
}
if (!$this->checkIfRunning()) {
} else {
return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.';
}
return true;
Expand All @@ -237,10 +194,96 @@ public function restartServer()
return 'Could not kill the previous instance of the ZeroMQ script.';
}
}
$this->__setupPubServer();
if (!is_numeric($this->checkIfRunning())) {
$settings = $this->getSetSettings();
$this->setupPubServer($settings);
if ($this->checkIfRunning() === false) {
return 'Failed starting the ZeroMQ script.';
}
return true;
}

/**
* @param array $settings
* @throws Exception
*/
private function setupPubServer(array $settings)
{
if ($this->checkIfRunning() === false) {
if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
// Old version is running, kill it and start again new one.
$redis = $this->createRedisConnection($settings);
$redis->rPush($settings['redis_namespace'] . ':command', 'kill');
sleep(1);
}

$this->saveSettingToFile($settings);
$server = ClassRegistry::init('Server');
shell_exec($server->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'mispzmq' . DS . 'mispzmq.py >> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.log 2>> ' . APP . 'tmp' . DS . 'logs' . DS . 'mispzmq.error.log &');
}
}

private function pushToRedis($ns, $data)
{
$this->redis->rPush($this->settings['redis_namespace'] . $ns, $data);
return true;
}

/**
* @param array $settings
* @return Redis
*/
private function createRedisConnection(array $settings)
{
$redis = new Redis();
$redis->connect($settings['redis_host'], $settings['redis_port']);
$redisPassword = $settings['redis_password'];
if (!empty($redisPassword)) {
$redis->auth($redisPassword);
}
$redis->select($settings['redis_database']);
return $redis;
}

/**
* @param array $settings
* @throws Exception
*/
private function saveSettingToFile(array $settings)
{
$settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json';
$settingsFile = new File($settingFilePath, true, 0644);
if (!$settingsFile->exists()) {
throw new Exception("Could not create zmq config file '$settingFilePath'.");
}
// Because setting file contains secrets, it should be readable just by owner. But because in Travis test,
// config file is created under one user and then changed under other user, file must be readable and writable
// also by group.
chmod($settingsFile->pwd(), 0660);
if (!$settingsFile->write(json_encode($settings))) {
throw new Exception("Could not write zmq config file '$settingFilePath'.");
}
$settingsFile->close();
}

private function getSetSettings()
{
$settings = array(
'redis_host' => 'localhost',
'redis_port' => '6379',
'redis_password' => '',
'redis_database' => '1',
'redis_namespace' => 'mispq',
'port' => '50000',
'username' => null,
'password' => null,
);

foreach ($settings as $key => $setting) {
$temp = Configure::read('Plugin.ZeroMQ_' . $key);
if ($temp) {
$settings[$key] = $temp;
}
}
return $settings;
}
}
4,681 changes: 2,350 additions & 2,331 deletions app/Locale/zh-s/LC_MESSAGES/default.po
View file

Large diffs are not rendered by default.

24 changes: 12 additions & 12 deletions app/Model/AppModel.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ class AppModel extends Model
{
public $name;

public $loadedPubSubTool = false;
/**
* @var PubSubTool
*/
private $loadedPubSubTool;

public $loadedKafkaPubTool = false;

Expand Down Expand Up @@ -78,7 +81,7 @@ public function __construct($id = false, $table = null, $ds = null)
33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false,
51 => false, 52 => false, 53 => false
51 => false, 52 => false, 53 => false, 54 => false
);

public $advanced_updates_description = array(
Expand Down Expand Up @@ -1389,6 +1392,9 @@ public function updateDatabase($command)
$this->__addIndex('user_settings', 'setting');
}
break;
case 54:
$sqlArray[] = "ALTER TABLE `sightingdbs` MODIFY `timestamp` int(11) NOT NULL DEFAULT 0;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
Expand Down Expand Up @@ -2364,20 +2370,14 @@ public function publishKafkaNotification($topicName, $data, $action = false) {
public function getPubSubTool()
{
if (!$this->loadedPubSubTool) {
$this->loadPubSubTool();
App::uses('PubSubTool', 'Tools');
$pubSubTool = new PubSubTool();
$pubSubTool->initTool();
$this->loadedPubSubTool = $pubSubTool;
}
return $this->loadedPubSubTool;
}

public function loadPubSubTool()
{
App::uses('PubSubTool', 'Tools');
$pubSubTool = new PubSubTool();
$pubSubTool->initTool();
$this->loadedPubSubTool = $pubSubTool;
return true;
}

public function getElasticSearchTool()
{
if (!$this->elasticSearchClient) {
Expand Down
151 changes: 65 additions & 86 deletions app/Model/Attribute.php
View file

Large diffs are not rendered by default.

85 changes: 51 additions & 34 deletions app/Model/Event.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,24 +173,25 @@ class Event extends AppModel
);

public $validFormats = array(
'attack' => array('html', 'AttackExport', 'html'),
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
'cache' => array('txt', 'CacheExport', 'cache'),
'csv' => array('csv', 'CsvExport', 'csv'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'json' => array('json', 'JsonExport', 'json'),
'netfilter' => array('txt', 'NetfilterExport', 'sh'),
'opendata' => array('txt', 'OpendataExport', 'txt'),
'openioc' => array('xml', 'OpeniocExport', 'ioc'),
'xml' => array('xml', 'XmlExport', 'xml'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'rpz' => array('txt', 'RPZExport', 'rpz'),
'text' => array('text', 'TextExport', 'txt'),
'hashes' => array('txt', 'HashesExport', 'txt'),
'csv' => array('csv', 'CsvExport', 'csv'),
'snort' => array('txt', 'NidsSnortExport', 'rules'),
'stix' => array('xml', 'Stix1Export', 'xml'),
'stix-json' => array('json', 'Stix1Export', 'json'),
'stix2' => array('json', 'Stix2Export', 'json'),
'suricata' => array('txt', 'NidsSuricataExport', 'rules'),
'text' => array('text', 'TextExport', 'txt'),
'xml' => array('xml', 'XmlExport', 'xml'),
'yara' => array('txt', 'YaraExport', 'yara'),
'yara-json' => array('json', 'YaraExport', 'json'),
'cache' => array('txt', 'CacheExport', 'cache'),
'attack' => array('html', 'AttackExport', 'html'),
'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
'netfilter' => array('txt', 'NetfilterExport', 'sh')
'yara-json' => array('json', 'YaraExport', 'json')
);

public $csv_event_context_fields_to_fetch = array(
Expand Down Expand Up @@ -1658,7 +1659,8 @@ public function filterEventIds($user, &$params = array(), &$result_count = 0)
'publish_timestamp' => array('function' => 'set_filter_timestamp', 'pop' => true),
'org' => array('function' => 'set_filter_org', 'pop' => true),
'uuid' => array('function' => 'set_filter_uuid', 'pop' => true),
'published' => array('function' => 'set_filter_published', 'pop' => true)
'published' => array('function' => 'set_filter_published', 'pop' => true),
'threat_level_id' => array('function' => 'set_filter_threat_level_id', 'pop' => true)
),
'Object' => array(
'object_name' => array('function' => 'set_filter_object_name'),
Expand Down Expand Up @@ -2230,6 +2232,8 @@ public function fetchEvent($user, $options = array(), $useCache = false)
}
$event = $this->__filterBlockedAttributesByTags($event, $options, $user);
$event['Attribute'] = $this->__attachSharingGroups(!$options['sgReferenceOnly'], $event['Attribute'], $sharingGroupData);

$proposalBlockAttributes = Configure::read('MISP.proposals_block_attributes');
// move all object attributes to a temporary container
$tempObjectAttributeContainer = array();
foreach ($event['Attribute'] as $key => $attribute) {
Expand Down Expand Up @@ -2284,10 +2288,7 @@ public function fetchEvent($user, $options = array(), $useCache = false)
}
}
}
if (
Configure::read('MISP.proposals_block_attributes') &&
!empty($options['allow_proposal_blocking'])
) {
if ($proposalBlockAttributes && !empty($options['allow_proposal_blocking'])) {
foreach ($results[$eventKey]['Attribute'][$key]['ShadowAttribute'] as $sa) {
if ($sa['proposal_to_delete'] || $sa['to_ids'] == 0) {
unset($results[$eventKey]['Attribute'][$key]);
Expand Down Expand Up @@ -2321,15 +2322,15 @@ public function fetchEvent($user, $options = array(), $useCache = false)
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit);
}
}
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
$this->Feed = ClassRegistry::init('Feed');
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
if (!empty($options['includeServerCorrelations']) && $user['org_id'] == Configure::read('MISP.host_org_id')) {
$this->Feed = ClassRegistry::init('Feed');
if (!empty($options['overrideLimit'])) {
$overrideLimit = true;
} else {
$overrideLimit = false;
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
$event['ShadowAttribute'] = $this->Feed->attachFeedCorrelations($event['ShadowAttribute'], $user, $event['Event'], $overrideLimit, 'Server');
}
if (empty($options['metadata'])) {
$this->Sighting = ClassRegistry::init('Sighting');
Expand Down Expand Up @@ -2715,6 +2716,14 @@ public function set_filter_published(&$params, $conditions, $options)
return $conditions;
}

public function set_filter_threat_level_id(&$params, $conditions, $options)
{
if (isset($params['threat_level_id'])) {
$conditions['AND']['Event.threat_level_id'] = $params['threat_level_id'];
}
return $conditions;
}

public function set_filter_tags(&$params, $conditions, $options)
{
if (!empty($params['tags'])) {
Expand Down Expand Up @@ -3722,7 +3731,7 @@ public function _add(&$data, $fromXml, $user, $org_id = 0, $passAlong = null, $f
}
}

public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null)
public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null, $force = false)
{
$data = $this->cleanupEventArrayFromXML($data);
unset($this->Attribute->validate['event_id']);
Expand Down Expand Up @@ -3760,7 +3769,7 @@ public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null)
// Conditions affecting all:
// user.org == event.org
// edit timestamp newer than existing event timestamp
if (!isset($data['Event']['timestamp']) || $data['Event']['timestamp'] > $existingEvent['Event']['timestamp']) {
if ($force || !isset($data['Event']['timestamp']) || $data['Event']['timestamp'] > $existingEvent['Event']['timestamp']) {
if (!isset($data['Event']['timestamp'])) {
$data['Event']['timestamp'] = $date;
}
Expand Down Expand Up @@ -3832,7 +3841,7 @@ public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null)
if (isset($data['Event']['Attribute'])) {
$data['Event']['Attribute'] = array_values($data['Event']['Attribute']);
foreach ($data['Event']['Attribute'] as $k => $attribute) {
$result = $this->Attribute->editAttribute($attribute, $this->id, $user, 0, $this->Log);
$result = $this->Attribute->editAttribute($attribute, $this->id, $user, 0, $this->Log, $force);
if ($result !== true) {
$validationErrors['Attribute'][] = $result;
}
Expand All @@ -3841,15 +3850,15 @@ public function _edit(&$data, $user, $id, $jobId = null, $passAlong = null)
if (isset($data['Event']['Object'])) {
$data['Event']['Object'] = array_values($data['Event']['Object']);
foreach ($data['Event']['Object'] as $k => $object) {
$result = $this->Object->editObject($object, $this->id, $user, $this->Log);
$result = $this->Object->editObject($object, $this->id, $user, $this->Log, $force);
if ($result !== true) {
$validationErrors['Object'][] = $result;
}
}
foreach ($data['Event']['Object'] as $object) {
if (isset($object['ObjectReference'])) {
foreach ($object['ObjectReference'] as $objectRef) {
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user, $this->Log);
$result = $this->Object->ObjectReference->captureReference($objectRef, $this->id, $user, $this->Log, $force);
}
}
}
Expand Down Expand Up @@ -6671,6 +6680,10 @@ public function restSearch($user, $returnFormat, $filters, $paramsOnly = false,
$this->Job->id = $jobId;
}

if (!empty($exportTool->use_default_filters)) {
$exportTool->setDefaultFilters($filters);
}

if (empty($exportTool->non_restrictive_export)) {
if (!isset($filters['to_ids'])) {
$filters['to_ids'] = 1;
Expand Down Expand Up @@ -6710,11 +6723,15 @@ public function restSearch($user, $returnFormat, $filters, $paramsOnly = false,
$subqueryElements = $this->harvestSubqueryElements($filters);
$filters = $this->addFiltersFromSubqueryElements($filters, $subqueryElements);

$filters['include_attribute_count'] = 1;
$eventid = $this->filterEventIds($user, $filters, $elementCounter);
$eventCount = count($eventid);
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
unset($eventid);
if (empty($exportTool->mock_query_only)) {
$filters['include_attribute_count'] = 1;
$eventid = $this->filterEventIds($user, $filters, $elementCounter);
$eventCount = count($eventid);
$eventids_chunked = $this->__clusterEventIds($exportTool, $eventid);
unset($eventid);
} else {
$eventids_chunked = array();
}
if (!empty($exportTool->additional_params)) {
$filters = array_merge($filters, $exportTool->additional_params);
}
Expand Down
244 changes: 138 additions & 106 deletions app/Model/Feed.php
View file

Large diffs are not rendered by default.

23 changes: 8 additions & 15 deletions app/Model/FuzzyCorrelateSsdeep.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,26 +67,19 @@ public function get_all_7_char_chunks($hash)

public function query_ssdeep_chunks($hash, $attribute_id)
{
//'12288:CeqW86Tf7xglFIV/4Zf8FkKBPFrmtJxv/znLABkeGevRcAqn9LqgqmlrexDvBIRF:CV6fxg7IeEOKXrmtJx3rLABk1eFElree'
$chunks = $this->ssdeep_prepare($hash);
// if chunk_size == chunk_size_1 OR 2*chunk_size == chunk_size
// SELECT * from ssdeep_chunks where (chunk_size = $chunk_size OR chunk_size*2 = $chunk_size)
// AND chunk_size

// Original algo from article https://www.virusbulletin.com/virusbulletin/2015/11/optimizing-ssdeep-use-scale
// also propose to insert chunk size to database, but current database schema doesn't contain that column.
// This optimisation can be add in future versions.
$result = $this->find('list', array(
'conditions' => array(
'AND' => array(
'OR' => array(
'FuzzyCorrelateSsdeep.chunk_size' => $chunks[0],
'FuzzyCorrelateSsdeep.chunk_size' => $chunks[0] * 2,
),
'OR' => array(
'FuzzyCorrelateSsdeep.chunk' => $chunks[1],
'FuzzyCorrelateSsdeep.chunk' => $chunks[2]
)
)
'FuzzyCorrelateSsdeep.chunk' => array_merge($chunks[1], $chunks[2]),
),
'fields' => array('FuzzyCorrelateSsdeep.attribute_id', 'FuzzyCorrelateSsdeep.attribute_id')
'fields' => array('FuzzyCorrelateSsdeep.attribute_id', 'FuzzyCorrelateSsdeep.attribute_id'),
'group' => 'FuzzyCorrelateSsdeep.attribute_id', // remove duplicates at database side
));

$to_save = array();
foreach (array(1, 2) as $type) {
foreach ($chunks[$type] as $chunk) {
Expand Down
8 changes: 5 additions & 3 deletions app/Model/Log.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ class Log extends AppModel
'enable',
'error',
'export',
'failed_registration',
'file_upload',
'galaxy',
'include_formula',
Expand All @@ -52,13 +51,15 @@ class Log extends AppModel
'pull',
'purge_events',
'push',
'registration',
'registration_error',
'remove_dead_workers',
'request',
'request_delegation',
'reset_auth_key',
'send_mail',
'security',
'serverSettingsEdit',
'succeeded_registration',
'tag',
'undelete',
'update',
Expand Down Expand Up @@ -195,7 +196,8 @@ public function createLogEntry($user, $action, $model, $modelId = 0, $title = ''
if (is_array($change)) {
$output = array();
foreach ($change as $field => $values) {
if (strpos($field, 'password') !== false) { // if field name contains password, replace value with asterisk
$isSecret = strpos($field, 'password') !== false || ($field === 'authkey' && Configure::read('Security.do_not_log_authkeys'));
if ($isSecret) {
$oldValue = $newValue = "*****";
} else {
list($oldValue, $newValue) = $values;
Expand Down
70 changes: 44 additions & 26 deletions app/Model/MispObject.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -506,15 +506,23 @@ public function fetchObjects($user, $options = array())
)
),
);
if (empty($options['includeAllTags'])) {
if (!empty($options['metadata'])) {
unset($params['contain']['Attribute']);
}
if (empty($options['metadata']) && empty($options['includeAllTags'])) {
$params['contain']['Attribute']['AttributeTag']['Tag']['conditions']['exportable'] = 1;
}
if (isset($options['contain'])) {
$params['contain'] = array_merge_recursive($params['contain'], $options['contain']);
} else {
$option['contain']['Event']['fields'] = array('id', 'info', 'org_id', 'orgc_id');
}
if (Configure::read('MISP.proposals_block_attributes') && isset($options['conditions']['AND']['Attribute.to_ids']) && $options['conditions']['AND']['Attribute.to_ids'] == 1) {
if (
empty($options['metadata']) &&
Configure::read('MISP.proposals_block_attributes') &&
isset($options['conditions']['AND']['Attribute.to_ids']) &&
$options['conditions']['AND']['Attribute.to_ids'] == 1
) {
$this->Attribute->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
$proposalRestriction = array(
'ShadowAttribute' => array(
Expand Down Expand Up @@ -544,7 +552,7 @@ public function fetchObjects($user, $options = array())
if (!isset($options['enforceWarninglist'])) {
$options['enforceWarninglist'] = false;
}
if (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted']) {
if (empty($options['metadata']) && (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted'])) {
$params['contain']['Attribute']['conditions']['AND']['Attribute.deleted'] = 0;
}
if (isset($options['group'])) {
Expand All @@ -566,23 +574,25 @@ public function fetchObjects($user, $options = array())
}
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $key => $objects) {
foreach ($objects as $key2 => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
unset($results[$key][$key2]);
continue;
}
if ($proposals_block_attributes) {
if (!empty($attribute['ShadowAttribute'])) {
if (empty($options['metadata'])) {
foreach ($results as $key => $objects) {
foreach ($objects as $key2 => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
unset($results[$key][$key2]);
} else {
unset($results[$key][$key2]['ShadowAttribute']);
continue;
}
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key][$key2]['Attribute']['data'] = $encodedFile;
if ($proposals_block_attributes) {
if (!empty($attribute['ShadowAttribute'])) {
unset($results[$key][$key2]);
} else {
unset($results[$key][$key2]['ShadowAttribute']);
}
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key][$key2]['Attribute']['data'] = $encodedFile;
}
}
}
}
Expand Down Expand Up @@ -947,7 +957,7 @@ public function captureObject($object, $eventId, $user, $log = false)
return 'fail';
}

public function editObject($object, $eventId, $user, $log)
public function editObject($object, $eventId, $user, $log, $force = false)
{
$object['event_id'] = $eventId;
if (isset($object['uuid'])) {
Expand All @@ -973,7 +983,7 @@ public function editObject($object, $eventId, $user, $log)
return true;
}
if (isset($object['timestamp'])) {
if ($existingObject['Object']['timestamp'] >= $object['timestamp']) {
if ($force || $existingObject['Object']['timestamp'] >= $object['timestamp']) {
return true;
}
} else {
Expand Down Expand Up @@ -1023,7 +1033,7 @@ public function editObject($object, $eventId, $user, $log)
}
if (!empty($object['Attribute'])) {
foreach ($object['Attribute'] as $attribute) {
$result = $this->Attribute->editAttribute($attribute, $eventId, $user, $object['id'], $log);
$result = $this->Attribute->editAttribute($attribute, $eventId, $user, $object['id'], $log, $force);
}
}
return true;
Expand Down Expand Up @@ -1344,7 +1354,8 @@ public function restSearch($user, $returnFormat, $filters, $paramsOnly = false,
'includeCorrelations' => !empty($filters['includeCorrelations']) ? $filters['includeCorrelations'] : 0,
'includeDecayScore' => !empty($filters['includeDecayScore']) ? $filters['includeDecayScore'] : 0,
'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0,
'metadata' => !empty($filters['metadata']) ? $filters['metadata'] : 0,
);
if (!empty($filters['attackGalaxy'])) {
$params['attackGalaxy'] = $filters['attackGalaxy'];
Expand Down Expand Up @@ -1377,6 +1388,9 @@ public function restSearch($user, $returnFormat, $filters, $paramsOnly = false,
if (!empty($filters['score'])) {
$params['score'] = $filters['score'];
}
if (!empty($filters['metadata'])) {
$params['metadata'] = $filters['metadata'];
}
if ($paramsOnly) {
return $params;
}
Expand Down Expand Up @@ -1423,8 +1437,16 @@ private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool
{
$continue = true;
while ($continue) {
$temp = '';
$this->Whitelist = ClassRegistry::init('Whitelist');
$results = $this->fetchObjects($user, $params, $continue);
if (empty($results)) {
$loop = false;
return true;
}
if ($elementCounter !== 0 && !empty($results)) {
$temp .= $exportTool->separator($exportToolParams);
}
if ($params['includeSightingdb']) {
$this->Sightingdb = ClassRegistry::init('Sightingdb');
$results = $this->Sightingdb->attachToObjects($results, $user);
Expand All @@ -1433,7 +1455,6 @@ private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool
$results = $this->Whitelist->removeWhitelistedFromArray($results, true);
$results = array_values($results);
$i = 0;
$temp = '';
foreach ($results as $object) {
$elementCounter++;
$handlerResult = $exportTool->handler($object, $exportToolParams);
Expand All @@ -1448,9 +1469,6 @@ private function __iteratedFetch($user, &$params, &$loop, &$tmpfile, $exportTool
if (!$loop) {
$continue = false;
}
if ($continue) {
$temp .= $exportTool->separator($exportToolParams);
}
fwrite($tmpfile, $temp);
}
return true;
Expand Down
3 changes: 2 additions & 1 deletion app/Model/Role.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ class Role extends AppModel
'id' => 'RolePermAuth',
'text' => 'Auth key access',
'readonlyenabled' => true,
'title' => 'Users with this permission have access to authenticating via their Auth keys, granting them access to the API.'
'title' => 'Users with this permission have access to authenticating via their Auth keys, granting them access to the API.',
'site_admin_optional' => true
),
'perm_regexp_access' => array(
'id' => 'RolePermRegexpAccess',
Expand Down
73 changes: 53 additions & 20 deletions app/Model/Server.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1261,6 +1261,15 @@ public function __construct($id = false, $table = null, $ds = null)
'type' => 'boolean',
'null' => true
),
'do_not_log_authkeys' => array(
'level' => 0,
'description' => __('If enabled, any authkey will be replaced by asterisks in Audit log.'),
'value' => false,
'errorMessage' => '',
'test' => 'testBool',
'type' => 'boolean',
'null' => true
),
'email_otp_enabled' => array(
'level'=> 2,
'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'),
Expand Down Expand Up @@ -1806,6 +1815,24 @@ public function __construct($id = false, $table = null, $ds = null)
'type' => 'numeric',
'afterHook' => 'zmqAfterHook',
),
'ZeroMQ_username' => array(
'level' => 2,
'description' => __('The username that client need to use to connect to ZeroMQ.'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
'afterHook' => 'zmqAfterHook',
),
'ZeroMQ_password' => array(
'level' => 2,
'description' => __('The password that client need to use to connect to ZeroMQ.'),
'value' => '',
'errorMessage' => '',
'test' => 'testForEmpty',
'type' => 'string',
'afterHook' => 'zmqAfterHook',
),
'ZeroMQ_redis_host' => array(
'level' => 2,
'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'),
Expand Down Expand Up @@ -2364,11 +2391,11 @@ public function beforeSave($options = array())
return true;
}

private function __getEventIdListBasedOnPullTechnique($technique, $server)
private function __getEventIdListBasedOnPullTechnique($technique, $server, $force = false)
{
if ("full" === $technique) {
// get a list of the event_ids on the server
$eventIds = $this->getEventIdsFromServer($server);
$eventIds = $this->getEventIdsFromServer($server, false, null, false, false, 'events', $force);
if ($eventIds === 403) {
return array('error' => array(1, null));
} elseif (is_string($eventIds)) {
Expand All @@ -2380,7 +2407,7 @@ private function __getEventIdListBasedOnPullTechnique($technique, $server)
$eventIds = array_reverse($eventIds);
}
} elseif ("update" === $technique) {
$eventIds = $this->getEventIdsFromServer($server, false, null, true, true);
$eventIds = $this->getEventIdsFromServer($server, false, null, true, true, 'events', $force);
if ($eventIds === 403) {
return array('error' => array(1, null));
} elseif (is_string($eventIds)) {
Expand Down Expand Up @@ -2475,7 +2502,7 @@ private function __checkIfEventSaveAble($event) {
return false;
}

private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId)
private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId, $force = false)
{
// check if the event already exist (using the uuid)
$existingEvent = $eventModel->find('first', array('conditions' => array('Event.uuid' => $event['Event']['uuid'])));
Expand All @@ -2492,7 +2519,7 @@ private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$su
if (!$existingEvent['Event']['locked'] && !$server['Server']['internal']) {
$fails[$eventId] = __('Blocked an edit to an event that was created locally. This can happen if a synchronised event that was created on this instance was modified by an administrator on the remote side.');
} else {
$result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong);
$result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force);
if ($result === true) {
$successes[] = $eventId;
} elseif (isset($result['error'])) {
Expand All @@ -2504,7 +2531,7 @@ private function __checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, &$su
}
}

private function __pullEvent($eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId)
private function __pullEvent($eventId, &$successes, &$fails, $eventModel, $server, $user, $jobId, $force = false)
{
$event = $eventModel->downloadEventFromServer(
$eventId,
Expand All @@ -2519,7 +2546,7 @@ private function __pullEvent($eventId, &$successes, &$fails, $eventModel, $serve
if (!$this->__checkIfEventSaveAble($event)) {
$fails[$eventId] = __('Empty event detected.');
} else {
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $server, $user, $jobId);
$this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $server, $user, $jobId, $force);
}
} else {
// error
Expand Down Expand Up @@ -2584,7 +2611,7 @@ private function __handlePulledProposals($proposals, $events, $job, $jobId, $eve
return $pulledProposals;
}

public function pull($user, $id = null, $technique=false, $server, $jobId = false)
public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false)
{
if ($jobId) {
$job = ClassRegistry::init('Job');
Expand All @@ -2598,7 +2625,7 @@ public function pull($user, $id = null, $technique=false, $server, $jobId = fals
$eventIds = array();
// if we are downloading a single event, don't fetch all proposals
$conditions = is_numeric($technique) ? array('Event.id' => $technique) : array();
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server);
$eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $server, $force);
$server['Server']['version'] = $this->getRemoteVersion($id);
if (!empty($eventIds['error'])) {
$errors = array(
Expand Down Expand Up @@ -2627,7 +2654,7 @@ public function pull($user, $id = null, $technique=false, $server, $jobId = fals
if (!empty($eventIds)) {
// download each event
foreach ($eventIds as $k => $eventId) {
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $server, $user, $jobId);
$this->__pullEvent($eventId, $successes, $fails, $eventModel, $server, $user, $jobId, $force);
if ($jobId) {
if ($k % 10 == 0) {
$job->saveField('progress', 50 * (($k + 1) / count($eventIds)));
Expand Down Expand Up @@ -2740,7 +2767,7 @@ private function __orgRuleDowngrade($HttpSocket, $request, $server, $filter_rule
}

// Get an array of event_ids that are present on the remote server
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events')
public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events', $force = false)
{
$url = $server['Server']['url'];
if ($ignoreFilterRules) {
Expand Down Expand Up @@ -2824,7 +2851,9 @@ public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $
}
}
}
$this->Event->removeOlder($eventArray, $scope);
if (!$force) {
$this->Event->removeOlder($eventArray, $scope);
}
if (!empty($eventArray)) {
foreach ($eventArray as $event) {
if ($force_uuid) {
Expand Down Expand Up @@ -4601,26 +4630,28 @@ private function __attachRecoveryQuery($field, $table)
switch($field['error_type']) {
case 'missing_column':
$field['sql'] = sprintf(
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s;',
'ALTER TABLE `%s` ADD COLUMN `%s` %s%s %s %s %s %s;',
$table,
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'],
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
);
break;
case 'column_different':
$field['sql'] = sprintf(
'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s;',
'ALTER TABLE `%s` MODIFY COLUMN `%s` %s%s %s %s %s %s;',
$table,
$field['column_name'],
$field['expected']['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($field['expected']['column_default']) ? 'DEFAULT "' . $field['expected']['column_default'] . '"' : '',
$field['expected']['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name']
empty($field['expected']['collation_name']) ? '' : 'COLLATE ' . $field['expected']['collation_name'],
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
);
break;
}
Expand All @@ -4637,13 +4668,14 @@ private function __attachRecoveryQuery($field, $table)
} elseif ($expectedField['data_type'] === 'text') {
$length = null;
}
$fieldSql = sprintf('`%s` %s%s %s %s %s',
$fieldSql = sprintf('`%s` %s%s %s %s %s %s',
$expectedField['column_name'],
$expectedField['data_type'],
$length !== null ? sprintf('(%d)', $length) : '',
isset($expectedField['column_default']) ? 'DEFAULT "' . $expectedField['column_default'] . '"' : '',
$expectedField['is_nullable'] === 'NO' ? 'NOT NULL' : 'NULL',
empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name']
empty($expectedField['collation_name']) ? '' : 'COLLATE ' . $expectedField['collation_name'],
empty($field['expected']['extra']) ? '' : $field['expected']['extra']
);
$allFields[] = $fieldSql;
}
Expand Down Expand Up @@ -4694,7 +4726,8 @@ public function getActualDBSchema(
// 'datetime_precision', -- Only available on MySQL 5.6+
'collation_name',
'column_type',
'column_default'
'column_default',
'extra',
)
){
$dbActualSchema = array();
Expand Down Expand Up @@ -4976,7 +5009,7 @@ public function yaraDiagnostics(&$diagnostic_errors)
public function stixDiagnostics(&$diagnostic_errors, &$stixVersion, &$cyboxVersion, &$mixboxVersion, &$maecVersion, &$stix2Version, &$pymispVersion)
{
$result = array();
$expected = array('stix' => '>1.2.0.6', 'cybox' => '>2.1.0.18.dev0', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.120');
$expected = array('stix' => '>1.2.0.9', 'cybox' => '>2.1.0.21', 'mixbox' => '1.0.3', 'maec' => '>4.1.0.14', 'stix2' => '>1.2.0', 'pymisp' => '>2.4.120');
// check if the STIX and Cybox libraries are working using the test script stixtest.py
$scriptResult = shell_exec($this->getPythonVersion() . ' ' . APP . 'files' . DS . 'scripts' . DS . 'stixtest.py');
$scriptResult = json_decode($scriptResult, true);
Expand Down
20 changes: 17 additions & 3 deletions app/Model/User.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,21 @@ private function __finaliseAndSendEmail($replyToUser, &$Email, &$replyToLog, $us
}
}
$Email->attachments($attachments);
$result = $Email->send($body);
try {
$result = $Email->send($body);
} catch (Exception $e) {
$this->Log = ClassRegistry::init('Log');
$this->Log->save(array(
'org' => 'SYSTEM',
'model' => 'User',
'model_id' => $user['User']['id'],
'email' => $user['User']['email'],
'action' => 'send_mail',
'title' => sprintf(__('Could not send mail. Reasons: %s'), $e->getMessage()),
'change' => null,
));
$result = false;
}
$Email->reset();
return $result;
}
Expand Down Expand Up @@ -1540,7 +1554,7 @@ public function registerUser($added_by, $registration, $org_id, $role_id) {
'model' => 'User',
'model_id' => $added_by['id'],
'email' => $added_by['email'],
'action' => 'failed_registration',
'action' => 'registration_error',
'title' => 'User registration failed for ' . $user['email'] . '. Reason(s): ' . $error,
'change' => null,
));
Expand All @@ -1555,7 +1569,7 @@ public function registerUser($added_by, $registration, $org_id, $role_id) {
'model' => 'User',
'model_id' => $added_by['id'],
'email' => $added_by['email'],
'action' => 'succeeded_registration',
'action' => 'registration',
'title' => sprintf('User registration success for %s (id=%s)', $user['User']['email'], $user['User']['id']),
'change' => null,
));
Expand Down
2 changes: 1 addition & 1 deletion app/Model/Warninglist.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public function getTldLists()
public function update()
{
$directories = glob(APP . 'files' . DS . 'warninglists' . DS . 'lists' . DS . '*', GLOB_ONLYDIR);
$updated = array();
$updated = array('success' => [], 'fails' => []);
foreach ($directories as $dir) {
$file = new File($dir . DS . 'list.json');
$list = json_decode($file->read(), true);
Expand Down
4 changes: 4 additions & 0 deletions app/Plugin/Assets/models/behaviors/LogableBehavior.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,10 @@ function afterSave(Model $Model, $created, $options = array()) {
$old = '';
}
if ($key != 'modified' && !in_array($key, $this->settings[$Model->alias]['ignore']) && $value != $old && in_array($key, $db_fields)) {
if ($key === 'authkey' && Configure::read('Security.do_not_log_authkeys')) {
$old = $value = '*****';
}

if ($this->settings[$Model->alias]['change'] == 'full') {
$changed_fields[] = $key . ' (' . $old . ') => (' . $value . ')';
} else if ($this->settings[$Model->alias]['change'] == 'serialize') {
Expand Down
8 changes: 6 additions & 2 deletions app/Plugin/SysLogLogable/Model/Behavior/SysLogLogableBehavior.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,18 @@ function afterSave(Model $Model, $created, $options = array()) {
if (isset($Model->data[$Model->alias][$Model->primaryKey]) && !empty($this->old) && isset($this->old[$Model->alias][$key])) {
$old = $this->old[$Model->alias][$key];
if (is_array($old)) {
$old = json_encode($old, true);
$old = json_encode($old);
}
} else {
$old = '';
}
// TODO Audit, removed 'revision' as well
if ($key != 'lastpushedid' && $key!= 'timestamp' && $key != 'revision' && $key != 'modified' && !in_array($key, $this->settings[$Model->alias]['ignore']) && $value != $old && in_array($key, $db_fields)) {
if ($this->settings[$Model->alias]['change'] == 'full') {
if ($key === 'authkey' && Configure::read('Security.do_not_log_authkeys')) {
$old = $value = '*****';
}

if ($this->settings[$Model->alias]['change'] == 'full') {
if (($key != 'published') || (($key == 'published') && ($value == '1'))) { // remove (un-)published from edit
$changed_fields[] = $key . ' (' . $old . ') => (' . $value . ')';
}
Expand Down
8 changes: 4 additions & 4 deletions app/View/Elements/Events/View/related_event.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,20 @@
<table>
<tbody>
<tr>
<td rowspan="2" style="border-right: 1px solid #ddd; padding-right: 2px; max-width: 24px; overflow: hidden; font-size: xx-small; text-overflow: ellipsis;" title="<?php echo h($related['Orgc']['name']); ?>">
<td rowspan="2" style="border-right: 1px solid #ddd; padding-right: 2px; min-width: 24px; max-width: 24px; overflow: hidden; font-size: xx-small; text-overflow: ellipsis;" title="<?php echo h($related['Orgc']['name']); ?>">
<?php echo $this->OrgImg->getOrgImg(array('name' => $related['Orgc']['name'], 'id' => $related['Orgc']['id'], 'size' => 24)); ?>
</td>
<td style="line-height: 14px; padding-left: 2px; white-space: nowrap; text-overflow: ellipsis; overflow: hidden; max-width: 430px;">
<td style="line-height: 14px; padding-left: 2px; white-space: nowrap; text-overflow: ellipsis; overflow: hidden; max-width: 410px;">
<a title="<?php echo h($related['info']); ?>" href="<?php echo h($href_url)?>">
<span><?php echo h($related['info']) ?>
<?php echo h($related['info']) ?>
</a>
</td>
</tr>
<tr>
<td style="line-height: 14px; padding-left: 2px;">
<i><?php echo h($related['date']); ?></i>
<?php if (isset($relatedEventCorrelationCount[$related['id']])): ?>
<b style="margin-left: 5px; float: right; cursor: help;" title="<?php echo sprintf(__('This related event contains %s unique correlation(s)'), h($relatedEventCorrelationCount[$related['id']])); ?>"> <?php echo h($relatedEventCorrelationCount[$related['id']]) ?></b>
<b style="margin-left: 5px; float: right; cursor: help;" title="<?php echo __('This related event contains %s unique correlation(s)', h($relatedEventCorrelationCount[$related['id']])); ?>"> <?php echo h($relatedEventCorrelationCount[$related['id']]) ?></b>
<?php endif; ?>
</td>
</tr>
Expand Down
34 changes: 17 additions & 17 deletions app/View/Elements/eventdiscussion.ctp
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
<h3><?= __('Discussion') ?></h3>
<div id="top">
<div class="pagination">
<?php
Expand All @@ -18,12 +19,11 @@
?>
</ul>
</div>
<div id = "posts">
<div id="posts">
<?php
foreach ($posts as $post) {
?>
<a name="message_<?php echo h($post['id']);?>"></a>
<table class="discussionBox" id=<?php echo '"' . h($post['id']) . '"';?> >
<table class="discussionBox" id="message_<?= h($post['id']) ?>">
<tr>
<td class="discussionBoxTD discussionBoxTDtop" colspan="2">
<div>
Expand All @@ -36,7 +36,7 @@
</td>
<td style="text-align:right">
<a href="#top" class="whitelink">Top</a> |
<a href="<?php echo "#".$post['id']; ?>" class="whitelink">#<?php echo h($post['id'])?></a>
<a href="#message_<?php echo h($post['id']); ?>" class="whitelink">#<?php echo h($post['id'])?></a>
</td>
</tr>
</table>
Expand All @@ -58,14 +58,14 @@
echo $this->Command->convertQuotes(nl2br(h($post['contents'])));
if ($post['post_id'] !=0 || ($post['date_created'] != $post['date_modified'])) {
?>
<br /><br />
<br><br>
<?php
}
if ($post['post_id'] != 0) {
?>
<span style="font-style:italic">
In reply to post
<a href="<?php echo "#".h($post['post_id']); ?>">#<?php echo h($post['post_id'])?></a>
<a href="#message_<?php echo h($post['post_id']); ?>">#<?php echo h($post['post_id'])?></a>
</span>
<?php
}
Expand All @@ -76,7 +76,7 @@
</td>
</tr>
<tr>
<td class="discussionBoxTD discussionBoxTDbottom" colspan = "2">
<td class="discussionBoxTD discussionBoxTDbottom" colspan="2">
<table style="width:100%">
<tr>
<td>
Expand All @@ -90,14 +90,14 @@
echo $this->Form->postLink('', array('controller' => 'posts', 'action' => 'delete', h($post['id']), h($context)), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete this post?'));
} else {
?>
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title = "<?php echo __('Reply');?>" aria-label = "<?php echo __('Reply');?>"></a>
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title="<?php echo __('Reply');?>" aria-label="<?php echo __('Reply');?>"></a>
<?php
}
} else {
echo $this->Html->link('', array('controller' => 'posts', 'action' => 'edit', h($post['id']), h($context)), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit')));
echo $this->Form->postLink('', array('controller' => 'posts', 'action' => 'delete', h($post['id']), h($context)), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete this post?'));
?>
<a href = "<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title = "<?php echo __('Reply');?>" aria-label = "<?php echo __('Reply');?>"></a>
<a href="<?php echo $baseurl.'/posts/add/post/'.h($post['id']); ?>" class="icon-comment" title="<?php echo __('Reply');?>" aria-label="<?php echo __('Reply');?>"></a>
<?php

}
Expand All @@ -108,7 +108,7 @@
</td>
</tr>
</table>
<br />
<br>
<?php
}
?>
Expand Down Expand Up @@ -139,11 +139,11 @@
?>
<fieldset>
<div class="input clear">
<button type="button" title="<?php echo __('Insert a quote - just paste your quote between the [quote][/quote] tags.');?>" class="toggle-left btn btn-inverse qet" id = "quote" onclick="insertQuote()"><?php echo __('Quote');?></button>
<button type="button" title="<?php echo __('Insert a link to an event - just enter the event ID between the [event][/event] tags.');?>" class="toggle btn btn-inverse qet" id = "event" onclick="insertEvent()"><?php echo __('Event');?></button>
<button type="button" title="<?php echo __('Insert a link to a discussion thread - enter the thread\'s ID between the [thread][/thread] tags.');?>" class="toggle btn btn-inverse qet" id = "thread" onclick="insertThread()"><?php echo __('Thread');?></button>
<button type="button" title="<?php echo __('Insert a link [link][/link] tags.');?>" class="toggle btn btn-inverse qet" id = "link" onclick="insertLink()"><?php echo __('Link');?></button>
<button type="button" title="<?php echo __('Insert a code [code][/code] tags.');?>" class="toggle-right btn btn-inverse qet" id = "code" onclick="insertCode()"><?php echo __('Code');?></button>
<button type="button" title="<?php echo __('Insert a quote - just paste your quote between the [quote][/quote] tags.');?>" class="toggle-left btn btn-inverse qet" id="quote" onclick="insertQuote()"><?php echo __('Quote');?></button>
<button type="button" title="<?php echo __('Insert a link to an event - just enter the event ID between the [event][/event] tags.');?>" class="toggle btn btn-inverse qet" id="event" onclick="insertEvent()"><?php echo __('Event');?></button>
<button type="button" title="<?php echo __('Insert a link to a discussion thread - enter the thread\'s ID between the [thread][/thread] tags.');?>" class="toggle btn btn-inverse qet" id="thread" onclick="insertThread()"><?php echo __('Thread');?></button>
<button type="button" title="<?php echo __('Insert a link [link][/link] tags.');?>" class="toggle btn btn-inverse qet" id="link" onclick="insertLink()"><?php echo __('Link');?></button>
<button type="button" title="<?php echo __('Insert a code [code][/code] tags.');?>" class="toggle-right btn btn-inverse qet" id="code" onclick="insertCode()"><?php echo __('Code');?></button>
</div>
<?php
echo $this->Form->input('message', array(
Expand All @@ -154,7 +154,7 @@
));
?>
</fieldset>
<button class="btn btn-primary" onClick="submitMessageForm('<?php echo $url;?>', 'PostViewForm', 'top'); return false;"><?php echo __('Send');?></button>
<button class="btn btn-primary" onclick="submitMessageForm('<?php echo $url;?>', 'PostViewForm', 'top'); return false;"><?php echo __('Send comment');?></button>
<?php
echo $this->Form->end();
?>
Expand All @@ -177,7 +177,7 @@
document.getElementById("PostMessage").value+="[Code][/Code]";
}
<?php if (isset($post_id) && $post_id): ?>
$(document).ready(function() {
$(function() {
location.hash = "#message_<?php echo h($post_id); ?>";
});
<?php endif; ?>
Expand Down
2 changes: 1 addition & 1 deletion app/View/Events/resolved_attributes.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
);
$popoverHTML = '';
foreach ($popover as $key => $popoverElement) {
$popoverHTML .= '<span class=\'bold\'>' . $key . '</span>: <span class=\'blue bold\'>' . $popoverElement . '</span><br />';
$popoverHTML .= '<span class=\'bold\'>' . $key . '</span>: <span class=\'blue bold\'>' . h($popoverElement) . '</span><br />';
}
?>
<a href="<?php echo $baseurl; ?>/events/view/<?php echo h($relation['Event']['id']);?>" data-toggle="popover" title="Attribute details" data-content="<?php echo h($popoverHTML); ?>" data-trigger="hover"><?php echo h($relation['Event']['id']);?></a>
Expand Down
4 changes: 2 additions & 2 deletions app/View/Events/show_i_o_c_results.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ if (0 != count($attributes)): ?>
<h4><?php echo __('Successfully added attributes');?>:</h4>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?php echo __('Uuid');?></th>
<th><?php echo __('UUID');?></th>
<th><?php echo __('Category');?></th>
<th><?php echo __('Type');?></th>
<th><?php echo __('Value');?></th>
Expand All @@ -29,7 +29,7 @@ if (isset($fails)):?>
<h4><?php echo __('Failed indicators');?>:</h4>
<table class="table table-striped table-hover table-condensed">
<tr>
<th><?php echo __('Uuid');?></th>
<th><?php echo __('UUID');?></th>
<th><?php echo __('Search term');?></th>
<th><?php echo __('Content');?></th>
</tr><?php
Expand Down
2 changes: 1 addition & 1 deletion app/View/Feeds/preview_event.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<div class="span8">
<h2><?php echo nl2br(h($title)); ?></h2>
<dl>
<dt><?php echo __('Uuid');?></dt>
<dt><?php echo __('UUID');?></dt>
<dd><?php echo h($event['Event']['uuid']); ?></dd>
<dt><?php echo Configure::read('MISP.showorgalternate') ? __('Source Organisation') : __('Org')?></dt>
<dd><?php echo h($event['Orgc']['name']); ?></dd>
Expand Down
30 changes: 14 additions & 16 deletions app/View/Helper/OrgImgHelper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,33 @@

// Helper to retrieve org images with the given parameters
class OrgImgHelper extends AppHelper {
const IMG_PATH = APP . WEBROOT_DIR . DS . 'img' . DS . 'orgs' . DS;

public function getOrgImg($options, $returnData = false, $raw = false) {
$imgPath = APP . WEBROOT_DIR . DS . 'img' . DS . 'orgs' . DS;
$imgOptions = array();
$possibleFields = array('id', 'name');
$size = !empty($options['size']) ? $options['size'] : 48;
foreach ($possibleFields as $field) {
if (isset($options[$field]) && file_exists($imgPath . $options[$field] . '.png')) {
if (isset($options[$field]) && file_exists(self::IMG_PATH . $options[$field] . '.png')) {
$imgOptions[$field] = $options[$field] . '.png';
break;
}
}
if (!empty($imgOptions)) {
foreach ($imgOptions as $field => $imgOption) {
if ($raw) {
$result = sprintf(
'<img src="/img/orgs/%s" title = "%s" style = "width: %spx; height: %spx;"/>',
$imgOption,
isset($options['name']) ? h($options['name']) : h($options['id']),
h($size),
h($size)
);
} else {
$result = sprintf(
'<img src="/img/orgs/%s" title="%s" width="%s" height="%s">',
$imgOption,
isset($options['name']) ? h($options['name']) : h($options['id']),
(int)$size,
(int)$size
);

if (!$raw) {
$result = sprintf(
'<a href="/organisations/view/%s"><img src="/img/orgs/%s" title = "%s" style = "width: %spx; height: %spx;"/></a>',
'<a href="/organisations/view/%s">%s</a>',
(empty($options['id']) ? h($options['name']) : h($options['id'])),
$imgOption,
isset($options['name']) ? h($options['name']) : h($options['id']),
h($size),
h($size)
$result
);
}
break;
Expand Down
2 changes: 1 addition & 1 deletion app/View/Organisations/index.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
<th><?php echo __('Logo');?></th>
<th><?php echo $this->Paginator->sort('name');?></th>
<?php if ($isSiteAdmin): ?>
<th><?php echo $this->Paginator->sort('uuid');?></th>
<th><?php echo $this->Paginator->sort('uuid', 'UUID');?></th>
<?php endif; ?>
<th><?php echo $this->Paginator->sort('description');?></th>
<th><?php echo $this->Paginator->sort('nationality');?></th>
Expand Down
2 changes: 1 addition & 1 deletion app/View/Pages/doc/using_the_system.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ Templates are devided into sections, with each section having a title and a desc
<b><?php echo __('General Event Information');?></b>
<ul>
<li><b><?php echo __('ID');?>:</b> <?php echo __('The ID of the event.');?></li>
<li><b><?php echo __('Uuid');?>:</b> <?php echo __('In order to avoid collisions between events and attributes (during for example a sync) a Uuid is assigned that uniquely identifies each of them.');?></li>
<li><b><?php echo __('UUID');?>:</b> <?php echo __('In order to avoid collisions between events and attributes (during for example a sync) a Uuid is assigned that uniquely identifies each of them.');?></li>
<li><b><?php echo __('Org');?></b> <?php echo __('The organisation that has originally created the event. The logo (if it exists on the server, alternatively a string) representing the organisation is also shown int he right upper corner.');?></li>
<li><b><?php echo __('Contributors');?>:</b> <?php echo __('Shows a list of the organisations that have contributed to the event via proposals. If you click any of the logos listed here, you\'ll get redirected to a filtered event history view, including only the changes made by the organisation.');?></li>
<li><b><?php echo __('Tags');?>:</b> <?php echo __('A list of tags associated with the event. Clicking a tag will show a list of events with the same tag attached. The little cross next to each tag allows you to remove the tag from the event, whilst the \'+\' button allows you to assign a tag. For the latter two options to be visible, you have to have tagging permission.');?></li>
Expand Down
6 changes: 5 additions & 1 deletion app/View/Roles/admin_add.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@
<?php
echo $this->Form->input($k, array(
'type' => 'checkbox',
'class' => 'checkbox ' . ($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
'class' => sprintf(
'checkbox %s %s',
($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
empty($flag['site_admin_optional']) ? 'site_admin_enforced' : 'site_admin_optional'
),
'checked' => false,
'label' => Inflector::humanize(substr($k, 5))
));
Expand Down
6 changes: 5 additions & 1 deletion app/View/Roles/admin_edit.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@
<?php
echo $this->Form->input($k, array(
'type' => 'checkbox',
'class' => 'checkbox ' . ($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
'class' => sprintf(
'checkbox %s %s',
($flag['readonlyenabled'] ? 'readonlyenabled' : 'readonlydisabled'),
empty($flag['site_admin_optional']) ? 'site_admin_enforced' : 'site_admin_optional'
),
'label' => Inflector::humanize(substr($k, 5))
));
if ($counter%3 == 0) echo "<div class = 'input clear'></div>";
Expand Down
2 changes: 1 addition & 1 deletion app/View/Servers/add.ctp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
<input type="text" id="ServerExternalName" <?php if (isset($this->request->data['Server']['external_name'])) echo 'value="' . $this->request->data['Server']['external_name'] . '"';?>>
</div>
<div id="ServerExternalUuidContainer" class="input select hiddenField" style="display:none;">
<label for="ServerExternalUuid"><?php echo __('Remote Organisation\'s Uuid');?></label>
<label for="ServerExternalUuid"><?php echo __('Remote Organisation\'s UUID');?></label>
<input type="text" id="ServerExternalUuid" <?php if (isset($this->request->data['Server']['external_uuid'])) echo 'value="' . $this->request->data['Server']['external_uuid'] . '"';?>>
</div>
<div class = "input clear"></div>
Expand Down
Loading