Merge branch '2.4' of github.com:MISP/MISP into 2.4

MISP · Jun 12, 2019 · 1716ca7 · 1716ca7
2 parents f2ea6b8 + 8d95ca0
commit 1716ca7
Show file tree

Hide file tree

Showing 60 changed files with 1,899 additions and 565 deletions.
diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh
@@ -1385,10 +1385,9 @@ coreCAKE () {
 updateGOWNT () {
   # AUTH_KEY Place holder in case we need to **curl** somehing in the future
   # 
-  # AUTH_KEY=$(mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP misp -e "SELECT authkey FROM users;" | tail -1)
-  # RHEL/CentOS
-  # AUTH_KEY=$(scl enable rh-mariadb102 "mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP misp -e 'SELECT authkey FROM users;' | tail -1")
-  #
+  $SUDO_WWW $RUN_MYSQL -- mysql -u $DBUSER_MISP -p$DBPASSWORD_MISP misp -e "SELECT authkey FROM users;" | tail -1 > /tmp/auth.key
+  AUTH_KEY=$(cat /tmp/auth.key)
+  rm /tmp/auth.key
 
   debug "Updating Galaxies, ObjectTemplates, Warninglists, Noticelists and Templates"
   # Update the galaxies…

diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv
@@ -1,5 +1,5 @@
-; Generated by RHash v1.3.8 on 2019-05-24 at 16:18.29
+; Generated by RHash v1.3.8 on 2019-05-30 at 14:36.25
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;        93945  16:18.29 2019-05-24 INSTALL.sh
-INSTALL.sh 792029108D63C94996B250AB3CF228D23BE05419 DF70E294BE6F496292B6228B187285D1132228052CD7F72B5FC0265CE97607C7 396D49CB2231383C50991DD585F6FC0DA214175D911AE5A54632B24CB4CA0C4BCC27ECA8CD690564E3E99232342D3ED9 3C0C5F4996F36BB810DA812DF80BB577F66A6F65BA9E1E14142929745F60680CB06E0C91913C90492D871F6B34DEEC75E1482F1D2BD827C605E026B65F8EBD79
+;        93871  14:36.25 2019-05-30 INSTALL.sh
+INSTALL.sh DE23B5D224757A8AB2941D8E15D73F10872D5106 ABEE81992478478406197EEC1891FA7CBDC5B32575447DD6865511B1DE48EC6F D7B9CA78779343C0CD47C9184DCA17DEFA24FA1B6BB35441F574AC40ED5A5AD68738BA91676E528AFC2488B44EC935C6 1FFDD293EF9FD53F80813B33839187ECF00B68FDDECA11327508ACABB99B45F45017CFF4AF2B70CF82D27B4AEEB75C34434B3AA00AD52D3AFC8405E77B8CF348
diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1
@@ -1 +1 @@
-792029108d63c94996b250ab3cf228d23be05419  INSTALL.sh
+de23b5d224757a8ab2941d8e15d73f10872d5106  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256
@@ -1 +1 @@
-df70e294be6f496292b6228b187285d1132228052cd7f72b5fc0265ce97607c7  INSTALL.sh
+abee81992478478406197eec1891fa7cbdc5b32575447dd6865511b1de48ec6f  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384
@@ -1 +1 @@
-396d49cb2231383c50991dd585f6fc0da214175d911ae5a54632b24cb4ca0c4bcc27eca8cd690564e3e99232342d3ed9  INSTALL.sh
+d7b9ca78779343c0cd47c9184dca17defa24fa1b6bb35441f574ac40ed5a5ad68738ba91676e528afc2488b44ec935c6  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512
@@ -1 +1 @@
-3c0c5f4996f36bb810da812df80bb577f66a6f65ba9e1e14142929745f60680cb06e0c91913c90492d871f6b34deec75e1482f1d2bd827c605e026b65f8ebd79  INSTALL.sh
+1ffdd293ef9fd53f80813b33839187ecf00b68fddeca11327508acabb99b45f45017cff4af2b70cf82d27b4aeeb75c34434b3aa00ad52d3afc8405e77b8cf348  INSTALL.sh
diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql
@@ -1269,13 +1269,13 @@ INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modi
 VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0);
 
 INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
-VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1);
+VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1);
 
 INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
-VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0);
+VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0);
 
 INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
-VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0);
+VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0);
 
 INSERT INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `default_role`)
 VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);

diff --git a/VERSION.json b/VERSION.json
@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":107}
+{"major":2, "minor":4, "hotfix":108}
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
@@ -46,10 +46,11 @@ class AppController extends Controller
 
     public $helpers = array('Utility', 'OrgImg', 'FontAwesome');
 
-    private $__queryVersion = '71';
+    private $__queryVersion = '76';
     public $pyMispVersion = '2.4.106';
     public $phpmin = '7.0';
     public $phprec = '7.2';
+    public $isApiAuthed = false;
 
     public $baseurl = '';
     public $sql_dump = false;
@@ -251,6 +252,7 @@ public function beforeFilter()
                             }
                             $this->Session->renew();
                             $this->Session->write(AuthComponent::$sessionKey, $user['User']);
+                            $this->isApiAuthed = true;
                         } else {
                             // User not authenticated correctly
                             // reset the session information
@@ -502,6 +504,9 @@ public function afterFilter()
             $this->Log = ClassRegistry::init('Log');
             echo json_encode($this->Log->getDataSource()->getLog(false, false), JSON_PRETTY_PRINT);
         }
+        if ($this->isApiAuthed && $this->_isRest()) {
+            session_destroy();
+        }
     }
 
     public function queryACL($debugType='findMissingFunctionNames', $content = false)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
@@ -1277,7 +1277,7 @@ public function delete($id, $hard = false)
         }
         if ($this->request->is('ajax')) {
             if ($this->request->is('post')) {
-                if ($this->__delete($id, $hard)) {
+                if ($this->Attribute->deleteAttribute($id, $this->Auth->user(), $hard)) {
                     return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Attribute deleted.')), 'status'=>200, 'type' => 'json'));
                 } else {
                     return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Attribute was not deleted.')), 'status'=>200, 'type' => 'json'));
@@ -1291,7 +1291,7 @@ public function delete($id, $hard = false)
             if (!$this->request->is('post') && !$this->_isRest()) {
                 throw new MethodNotAllowedException();
             }
-            if ($this->__delete($id, $hard)) {
+            if ($this->Attribute->deleteAttribute($id, $this->Auth->user(), $hard)) {
                 if ($this->_isRest() || $this->response->type() === 'application/json') {
                     $this->set('message', 'Attribute deleted.');
                     $this->set('_serialize', array('message'));
@@ -1360,77 +1360,6 @@ public function restore($id = null)
         }
     }
 
-
-    // unification of the actual delete for the multi-select
-    private function __delete($id, $hard = false)
-    {
-        $this->Attribute->id = $id;
-        if (!$this->Attribute->exists()) {
-            return false;
-        }
-        $result = $this->Attribute->find('first', array(
-            'conditions' => array('Attribute.id' => $id),
-            'fields' => array('Attribute.*'),
-            'contain' => array('Event' => array(
-                'fields' => array('Event.*')
-            )),
-        ));
-        if (empty($result)) {
-            throw new MethodNotAllowedException(__('Attribute not found or not authorised.'));
-        }
-
-        // check for permissions
-        if (!$this->_isSiteAdmin()) {
-            if ($result['Event']['locked']) {
-                if ($this->Auth->user('org_id') != $result['Event']['org_id'] || !$this->userRole['perm_sync']) {
-                    throw new MethodNotAllowedException(__('Attribute not found or not authorised.'));
-                }
-            } else {
-                if ($this->Auth->user('org_id') != $result['Event']['orgc_id']) {
-                    throw new MethodNotAllowedException(__('Attribute not found or not authorised.'));
-                }
-            }
-        }
-        $date = new DateTime();
-        if ($hard) {
-            $save = $this->Attribute->delete($id);
-        } else {
-            if (Configure::read('Security.sanitise_attribute_on_delete')) {
-                $result['Attribute']['category'] = 'Other';
-                $result['Attribute']['type'] = 'comment';
-                $result['Attribute']['value'] = 'deleted';
-                $result['Attribute']['comment'] = '';
-                $result['Attribute']['to_ids'] = 0;
-            }
-            $result['Attribute']['deleted'] = 1;
-            $result['Attribute']['timestamp'] = $date->getTimestamp();
-            $save = $this->Attribute->save($result);
-            $object_refs = $this->Attribute->Object->ObjectReference->find('all', array(
-                'conditions' => array(
-                    'ObjectReference.referenced_type' => 0,
-                    'ObjectReference.referenced_id' => $id,
-                ),
-                'recursive' => -1
-            ));
-            foreach ($object_refs as $ref) {
-                $ref['ObjectReference']['deleted'] = 1;
-                $this->Attribute->Object->ObjectReference->save($ref);
-            }
-        }
-        // attachment will be deleted with the beforeDelete() function in the Model
-        if ($save) {
-            // We have just deleted the attribute, let's also check if there are any shadow attributes that were attached to it and delete them
-            $this->loadModel('ShadowAttribute');
-            $this->ShadowAttribute->deleteAll(array('ShadowAttribute.old_id' => $id), false);
-
-            // remove the published flag from the event
-            $this->Attribute->Event->unpublishEvent($result['Event']['id']);
-            return true;
-        } else {
-            return false;
-        }
-    }
-
     public function deleteSelected($id = false, $hard = false)
     {
         if (!$this->request->is('post')) {
@@ -1497,11 +1426,11 @@ public function deleteSelected($id = false, $hard = false)
         $successes = array();
         foreach ($attributes as $a) {
             if ($hard) {
-                if ($this->__delete($a['Attribute']['id'], true)) {
+                if ($this->Attribute->deleteAttribute($a['Attribute']['id'], $this->Auth->user(), true)) {
                     $successes[] = $a['Attribute']['id'];
                 }
             } else {
-                if ($this->__delete($a['Attribute']['id'], $a['Attribute']['deleted'] == 1 ? true : false)) {
+                if ($this->Attribute->deleteAttribute($a['Attribute']['id'], $this->Auth->user(), $a['Attribute']['deleted'] == 1 ? true : false)) {
                     $successes[] = $a['Attribute']['id'];
                 }
             }

diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
@@ -75,6 +75,7 @@ class ACLComponent extends Component
                     'delete' => array(),
                     'edit' => array(),
                     'index' => array(),
+                    'massDelete' => array()
             ),
             'eventDelegations' => array(
                     'acceptDelegation' => array('perm_add'),
@@ -96,6 +97,7 @@ class ACLComponent extends Component
                     'create_dummy_event' => array(),
                     'create_massive_dummy_events' => array(),
                     'csv' => array('*'),
+                    'cullEmptyEvents' => array(),
                     'delegation_index' => array('*'),
                     'delete' => array('perm_add'),
                     'deleteNode' => array('*'),
@@ -249,6 +251,8 @@ class ACLComponent extends Component
                 'edit' => array('perm_add'),
                 'get_row' => array('perm_add'),
                 'orphanedObjectDiagnostics' => array(),
+                'proposeObjectsFromAttributes' => array('*'),
+                'groupAttributesIntoObject' => array('perm_add'),
                 'revise_object' => array('perm_add'),
                 'view' => array('*'),
             ),
@@ -327,6 +331,7 @@ class ACLComponent extends Component
                     'add' => array(),
                     'cache' => array('perm_site_admin'),
                     'checkout' => array(),
+                    'createSync' => array('perm_sync'),
                     'delete' => array(),
                     'deleteFile' => array(),
                     'edit' => array(),
@@ -339,6 +344,7 @@ class ACLComponent extends Component
                     'getSubmodulesStatus' => array('perm_site_admin'),
                     'getSubmoduleQuickUpdateForm' => array('perm_site_admin'),
                     'getVersion' => array('*'),
+                    'import' => ('perm_site_admin'),
                     'index' => array('OR' => array('perm_sync', 'perm_admin')),
                     'ondemandAction' => array(),
                     'postTest' => array('perm_sync'),

diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php
@@ -47,7 +47,7 @@ class RestResponseComponent extends Component
                     Besides the parameters listed, other, format specific ones can be passed along (for example: requested_attributes and includeContext for the CSV export).
                     This API allows pagination via the page and limit parameters.",
                 'mandatory' => array('returnFormat'),
-                'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo', 'includeProposals'),
+                'optional' => array('page', 'limit', 'value' , 'type', 'category', 'org', 'tags', 'date', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'includeEventTags', 'event_timestamp', 'threat_level_id', 'eventinfo', 'includeProposals'),
                 'params' => array()
             )
         ),
@@ -75,7 +75,7 @@ class RestResponseComponent extends Component
                     Besides the parameters listed, other, format specific ones can be passed along (for example: requested_attributes and includeContext for the CSV export).
                     This API allows pagination via the page and limit parameters.",
                 'mandatory' => array('returnFormat'),
-                'optional' => array('page', 'limit', 'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'eventinfo'),
+                'optional' => array('page', 'limit', 'value', 'type', 'category', 'org', 'tag', 'tags', 'searchall', 'date', 'last', 'eventid', 'withAttachments', 'metadata', 'uuid', 'published', 'publish_timestamp', 'timestamp', 'enforceWarninglist', 'sgReferenceOnly', 'eventinfo'),
                 'params' => array()
             )
         ),
@@ -431,9 +431,7 @@ private function __sendResponse($response, $code, $format = false, $raw = false,
                 $type = $format;
             }
             if (!$raw) {
-                if (!empty($response)) {
-                    $response = json_encode($response, JSON_PRETTY_PRINT);
-                }
+                $response = json_encode($response, JSON_PRETTY_PRINT);
             }
         }
         $cakeResponse = new CakeResponse(array('body'=> $response, 'status' => $code, 'type' => $type));
@@ -681,6 +679,7 @@ private function __setup()
                 'todayHighlight' => true,
                 'autoclose' => true
             ),
+            'help' => 'The user set date field on the event level. If you are using restSearch, you can use any of the valid time related filters (examples: 7d, timestamps, [14d, 7d] for ranges, etc.)'
         ),
         'datefrom' => array(
             'type' => 'date',
@@ -840,7 +839,7 @@ private function __setup()
                 'autoclose' => true
             ),
             'help' => 'The date from which the event was published'
-        ),
+         ),
         'gpgkey' => array(
             'input' => 'text',
             'type' => 'string',
@@ -1539,6 +1538,7 @@ private function __overwriteType($scope, &$field) {
                 break;
         }
     }
+
     private function __overwriteCategory($scope, &$field) {
         $field['values'] = array_keys(ClassRegistry::init("Attribute")->categoryDefinitions);
     }

diff --git a/app/Controller/EventBlacklistsController.php b/app/Controller/EventBlacklistsController.php
@@ -27,13 +27,25 @@ public function beforeFilter()
 
     public function index()
     {
+        $passedArgsArray = array();
+        $passedArgs = $this->passedArgs;
         $params = array();
-        $validParams = array('event_uuid', 'comment');
+        $validParams = array('event_uuid', 'comment', 'event_info', 'event_orgc');
         foreach ($validParams as $validParam) {
             if (!empty($this->params['named'][$validParam])) {
                 $params[$validParam] = $this->params['named'][$validParam];
             }
         }
+        if (!empty($this->params['named']['searchall'])) {
+            $params['AND']['OR'] = array(
+                'event_uuid' => $this->params['named']['searchall'],
+                'comment' => $this->params['named']['searchall'],
+                'event_info' => $this->params['named']['searchall'],
+                'event_orgc' => $this->params['named']['searchall']
+            );
+        }
+        $this->set('passedArgs', json_encode($passedArgs));
+        $this->set('passedArgsArray', $passedArgsArray);
         $this->BlackList->index($this->_isRest(), $params);
     }
 
@@ -51,4 +63,39 @@ public function delete($id)
     {
         $this->BlackList->delete($this->_isRest(), $id);
     }
+
+    public function massDelete()
+    {
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $ids = $this->request->data['EventBlacklist']['ids'];
+            $event_ids = json_decode($ids, true);
+            if (empty($event_ids)) {
+                throw new NotFoundException(__('Invalid event IDs.'));
+            }
+            $result = $this->EventBlacklist->deleteAll(array('EventBlacklist.id' => $event_ids));
+            if ($result) {
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('EventBlacklist', 'Deleted', $ids, $this->response->type());
+                } else {
+                    $this->Flash->success('Event deleted.');
+                    $this->redirect(array('controller' => 'eventBlacklists', 'action' => 'index'));
+                }
+            } else {
+                $error = __('Failed to delete Event from EventBlacklist. Error: ') . PHP_EOL . h($result);
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('EventBlacklist', 'Deleted', false, $error, $this->response->type());
+                } else {
+                    $this->Flash->error($error);
+                    $this->redirect(array('controller' => 'eventBlacklists', 'action' => 'index'));
+                }
+            }
+        } else {
+            $ids = json_decode($this->request->query('ids'), true);
+            if (empty($ids)) {
+                throw new NotFoundException(__('Invalid event IDs.'));
+
+            }
+            $this->set('event_ids', $ids);
+        }
+    }
 }