Merge branch 'hotfix/4.0.5' into master-th4

TheHive-Project · Feb 8, 2021 · 9a530bc · 9a530bc
2 parents e2c0b7e + 3092e1c
commit 9a530bc
Show file tree

Hide file tree

Showing 44 changed files with 666 additions and 386 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -8,7 +8,7 @@ steps:
   - name: submodules
     image: alpine/git
     commands:
-      - git submodule update --recursive --init --remote
+      - git submodule update --recursive --init
 
   # Restore cache of downloaded dependencies
   - name: restore-cache

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,34 @@
 # Change Log
 
+## [4.0.5](https://github.com/TheHive-Project/TheHive/milestone/68) (2021-02-08)
+
+**Implemented enhancements:**
+
+- Support for using asterisks by tag-filtering [\#933](https://github.com/TheHive-Project/TheHive/issues/933)
+- "Close tasks and case" deletes tasks instead of closing them [\#1755](https://github.com/TheHive-Project/TheHive/issues/1755)
+- [Enhancement] Add schema update status in status API [\#1782](https://github.com/TheHive-Project/TheHive/issues/1782)
+
+**Closed issues:**
+
+- Running TheHive 4.0.1-1 it appears that application.log is no longer rotated. [\#1746](https://github.com/TheHive-Project/TheHive/issues/1746)
+
+**Fixed bugs:**
+
+- [Bug] RPM package does not create secret.conf file [\#1248](https://github.com/TheHive-Project/TheHive/issues/1248)
+- [Bug] More webhooks or more detailed webhook events [\#1739](https://github.com/TheHive-Project/TheHive/issues/1739)
+- [Bug] Webhooks opening infinite amount of files [\#1743](https://github.com/TheHive-Project/TheHive/issues/1743)
+- [Bug] Dashboards are always created as private [\#1754](https://github.com/TheHive-Project/TheHive/issues/1754)
+- [Bug]/Unable to get MISP organisation [\#1758](https://github.com/TheHive-Project/TheHive/issues/1758)
+- [Bug] TheHive 4 Cluster and Haproxy with roundrobin [\#1760](https://github.com/TheHive-Project/TheHive/issues/1760)
+- [Bug] TheHive -> MISP works. MISP -> TheHive not. [\#1761](https://github.com/TheHive-Project/TheHive/issues/1761)
+- [Bug] TheHive 4.0.4 cannot show tasks created in previous versions [\#1763](https://github.com/TheHive-Project/TheHive/issues/1763)
+- [Bug] `Imported` property in Alerts not taken into account  [\#1769](https://github.com/TheHive-Project/TheHive/issues/1769)
+- [Bug] Sort field list in dashboard widget filters [\#1771](https://github.com/TheHive-Project/TheHive/issues/1771)
+- [Bug] Dashboard on organisation (and other) doesn't work [\#1772](https://github.com/TheHive-Project/TheHive/issues/1772)
+- [BUG] Cannot link multiple organisations together [\#1773](https://github.com/TheHive-Project/TheHive/issues/1773)
+- [Bug] Fix pivoting from donuts to search pages on custom fields based widgets [\#1777](https://github.com/TheHive-Project/TheHive/issues/1777)
+- [Bug] Fix custom field filters in v0 APIs [\#1779](https://github.com/TheHive-Project/TheHive/issues/1779)
+
 ## [4.0.4](https://github.com/TheHive-Project/TheHive/milestone/67) (2021-01-12)
 
 **Implemented enhancements:**
@@ -269,7 +298,6 @@
 - SearchSrv.NotFoundError  [\#1242](https://github.com/TheHive-Project/TheHive/issues/1242)
 - Assignee is not changeable [\#1243](https://github.com/TheHive-Project/TheHive/issues/1243)
 - [Bug] In TheHive, a user is a member of one or more organisations. One user has a profile for each organisation and can have different profiles for different organisations.  [\#1247](https://github.com/TheHive-Project/TheHive/issues/1247)
-- [Bug] RPM package does not create secret.conf file [\#1248](https://github.com/TheHive-Project/TheHive/issues/1248)
 - [Bug] Unable to save new or imported dashboards in 4.0-RC1 [\#1250](https://github.com/TheHive-Project/TheHive/issues/1250)
 - [Bug] Header Variable authentication does not work [\#1251](https://github.com/TheHive-Project/TheHive/issues/1251)
 - Filtering by custom fields returns no results [\#1252](https://github.com/TheHive-Project/TheHive/issues/1252)

diff --git a/ScalliGraph b/ScalliGraph
diff --git a/build.sbt b/build.sbt
@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.0.4-1"
+val thehiveVersion         = "4.0.5-1"
 val scala212               = "2.12.12"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)

diff --git a/conf/migration-logback.xml b/conf/migration-logback.xml
@@ -5,7 +5,7 @@
                     converterClass="play.api.libs.logback.ColoredLevel"/>
 
     <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>./logs/migration.log</file>
+        <file>${application.home:-.}/logs/migration.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
             <fileNamePattern>${application.home:-.}/logs/application.%i.log.zip</fileNamePattern>
             <minIndex>1</minIndex>

diff --git a/...ector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchemaDefinition.scala b/...ector/src/main/scala/org/thp/thehive/connector/cortex/models/CortexSchemaDefinition.scala
@@ -14,8 +14,7 @@ import scala.reflect.runtime.{universe => ru}
 class CortexSchemaDefinition @Inject() () extends Schema with UpdatableSchema {
 
   lazy val logger: Logger    = Logger(getClass)
-  val name: String           = "thehive-cortex"
-  val operations: Operations = Operations(name)
+  val operations: Operations = Operations("thehive-cortex")
 
   lazy val reflectionClasses = new Reflections(
     new ConfigurationBuilder()

diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Connector.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/Connector.scala
@@ -2,10 +2,13 @@ package org.thp.thehive.connector.cortex.services
 
 import akka.actor.ActorSystem
 import akka.stream.Materializer
+
 import javax.inject.{Inject, Singleton}
 import org.thp.cortex.client.{CortexClient, CortexClientConfig}
+import org.thp.scalligraph.models.SchemaStatus
 import org.thp.scalligraph.services.config.ApplicationConfig.finiteDurationFormat
 import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem}
+import org.thp.thehive.connector.cortex.models.CortexSchemaDefinition
 import org.thp.thehive.models.HealthStatus
 import org.thp.thehive.services.{Connector => TheHiveConnector}
 import play.api.libs.json.{JsObject, Json}
@@ -17,6 +20,7 @@ import scala.util.{Failure, Success}
 @Singleton
 class Connector @Inject() (
     appConfig: ApplicationConfig,
+    schemaDefinition: CortexSchemaDefinition,
     mat: Materializer,
     implicit val system: ActorSystem,
     implicit val ec: ExecutionContext
@@ -44,10 +48,11 @@ class Connector @Inject() (
       .traverse(clients)(_.getHealth)
       .foreach { healthStatus =>
         val distinctStatus = healthStatus.toSet.map(HealthStatus.withName)
-        cachedHealth = if (distinctStatus.contains(HealthStatus.Ok)) {
-          if (distinctStatus.size > 1) HealthStatus.Warning else HealthStatus.Ok
-        } else if (distinctStatus.contains(HealthStatus.Error)) HealthStatus.Error
-        else HealthStatus.Warning
+        cachedHealth =
+          if (distinctStatus.contains(HealthStatus.Ok))
+            if (distinctStatus.size > 1) HealthStatus.Warning else HealthStatus.Ok
+          else if (distinctStatus.contains(HealthStatus.Error)) HealthStatus.Error
+          else HealthStatus.Warning
 
         system.scheduler.scheduleOnce(statusCheckInterval)(updateHealth())
       }
@@ -67,9 +72,10 @@ class Connector @Inject() (
       }
       .foreach { statusDetails =>
         val distinctStatus = statusDetails.map(_._3).toSet
-        val healthStatus = if (distinctStatus.contains("OK")) {
-          if (distinctStatus.size > 1) "WARNING" else "OK"
-        } else "ERROR"
+        val healthStatus =
+          if (distinctStatus.contains("OK"))
+            if (distinctStatus.size > 1) "WARNING" else "OK"
+          else "ERROR"
 
         cachedStatus = Json.obj(
           "enabled" -> true,
@@ -83,4 +89,5 @@ class Connector @Inject() (
       }
   updateStatus()
 
+  override def schemaStatus: Option[SchemaStatus] = schemaDefinition.schemaStatus
 }
diff --git a/...ex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/TestConnector.scala b/...ex/connector/src/test/scala/org/thp/thehive/connector/cortex/services/TestConnector.scala
@@ -2,15 +2,23 @@ package org.thp.thehive.connector.cortex.services
 
 import akka.actor.ActorSystem
 import akka.stream.Materializer
+
 import javax.inject.{Inject, Singleton}
 import org.thp.cortex.client.CortexClient
 import org.thp.scalligraph.services.config.ApplicationConfig
+import org.thp.thehive.connector.cortex.models.CortexSchemaDefinition
 
 import scala.concurrent.ExecutionContext
 
 @Singleton
-class TestConnector @Inject() (client: CortexClient, appConfig: ApplicationConfig, mat: Materializer, system: ActorSystem, ec: ExecutionContext)
-    extends Connector(appConfig, mat, system, ec) {
+class TestConnector @Inject() (
+    client: CortexClient,
+    appConfig: ApplicationConfig,
+    schemaDefinition: CortexSchemaDefinition,
+    mat: Materializer,
+    system: ActorSystem,
+    ec: ExecutionContext
+) extends Connector(appConfig, schemaDefinition, mat, system, ec) {
   override def clients: Seq[CortexClient] = Seq(client)
 
   override protected def updateHealth(): Unit = ()

diff --git a/frontend/app/scripts/services/api/DashboardSrv.js b/frontend/app/scripts/services/api/DashboardSrv.js
@@ -233,6 +233,7 @@
                         _.each(metadata.entities, function(entity) {
                             metadata[entity] = _.omit(data[entity], 'attributes');
                             metadata[entity].attributes = self._objectifyBy(data[entity].attributes, 'name');
+                            metadata[entity].attributeKeys = _.keys(metadata[entity].attributes).sort();
                         });
 
                         self.metadata[version] = metadata;

diff --git a/frontend/app/scripts/services/common/QueryBuilderSrv.js b/frontend/app/scripts/services/common/QueryBuilderSrv.js
@@ -54,11 +54,47 @@
 
             if(values.length > 0) {
                 var criterions = _.map(values, function(val) {
-                    var v = {_like: {}};
+                    return {_like: {
+                        _field: filter.field,
+                        _value: val
+                    }};
+                });
 
-                    v._like[filter.field] = val;
+                var criteria = {};
+                switch(operator) {
+                    case 'all':
+                        criteria = criterions.length === 1 ? criterions[0] : { _and: criterions };
+                        break;
+                    case 'none':
+                        criteria = {
+                            _not: criterions.length === 1 ? criterions[0] : { _or: criterions }
+                        };
+                        break;
+                    default:
+                        criteria = criterions.length === 1 ? criterions[0] : { _or: criterions };
+                }
 
-                    return v;
+                return criteria;
+            }
+
+            return null;
+        };
+
+        this._buildQueryFromTagsFilter = function(fieldDef, filter) {
+            if (!filter || !filter.value) {
+                return null;
+            }
+            var operator = filter.value.operator || 'any';
+            var values = _.pluck(filter.value.list, 'text');
+
+            if(values.length > 0) {
+                var criterions = _.map(values, function(val) {
+                    return {
+                        _like: {
+                            _field: filter.field,
+                            _value: val
+                        }
+                    };
                 });
 
                 var criteria = {};
@@ -71,6 +107,7 @@
                             _not: criterions.length === 1 ? criterions[0] : { _or: criterions }
                         };
                         break;
+                    //case 'any':
                     default:
                         criteria = criterions.length === 1 ? criterions[0] : { _or: criterions };
                 }
@@ -167,6 +204,8 @@
                 return this._buildQueryFromDateFilter(fieldDef, filter);
             } else if(filter.type === 'boolean') {
                 return this._buildQueryFromBooleanFilter(fieldDef, filter);
+            } else if(filter.field === 'tags') {
+                return this._buildQueryFromTagsFilter(fieldDef, filter);
             } else if(filter.type === 'user' || filter.field === 'tags' || filter.type === 'enumeration') {
                 return this._buildQueryFromListFilter(fieldDef, filter);
             } else if(filter.type === 'string' && fieldDef.values.length === 0) {

diff --git a/frontend/app/scripts/services/ui/GlobalSearchSrv.js b/frontend/app/scripts/services/ui/GlobalSearchSrv.js
@@ -55,26 +55,37 @@
         };
 
         this.buildDefaultFilterValue = function(fieldDef, value) {
-            if(fieldDef.name === 'tags' || fieldDef.type === 'user' || fieldDef.values.length > 0) {
+
+            var valueId = value.id;
+            var valueName = value.name;
+
+            if(valueId.startsWith('"') && valueId.endsWith('"')) {
+                valueId = valueId.slice (1, valueId.length-1);
+            }
+            if(valueName.startsWith('"') && valueName.endsWith('"')) {
+                valueName = valueName.slice (1, valueName.length-1);
+            }
+
+            if(fieldDef.type === 'string' || fieldDef.name === 'tags' || fieldDef.type === 'user' || fieldDef.values.length > 0) {
                 return {
                     operator: 'any',
                     list: [{
-                        text: (fieldDef.type === 'number' || fieldDef.type === 'integer') ? Number.parseInt(value.id) : value.id, label:value.name
+                        text: (fieldDef.type === 'number' || fieldDef.type === 'integer') ? Number.parseInt(valueId) : valueId, label:valueName
                     }]
                 };
             } else {
                 switch(fieldDef.type) {
                     case 'number':
                     case 'integer':
                         return {
-                            value: Number.parseInt(value.id)
+                            value: Number.parseInt(valueId)
                         };
                     case 'boolean':
-                        return value.id === 'true';
+                        return valueId === 'true';
                     default:
-                      return value.id;
+                      return valueId;
                 }
-                return value.id;
+                return valueId;
             }
 
         };

diff --git a/frontend/app/views/directives/dashboard/filters.html b/frontend/app/views/directives/dashboard/filters.html
@@ -10,7 +10,7 @@
                 </button>
             </span>
             <select class="form-control" ng-model="filter.field"
-                ng-options="item.name as item.name for (key, item) in metadata[component.options.entity].attributes"
+                ng-options="item for item in metadata[component.options.entity].attributeKeys"
                 ng-change="setFilterField(filter, component.options.entity)"></select>
         </div>
     </div>

diff --git a/frontend/app/views/directives/dashboard/multiline/serie.filters.html b/frontend/app/views/directives/dashboard/multiline/serie.filters.html
@@ -10,7 +10,7 @@
                 </button>
             </span>
             <select class="form-control" ng-model="filter.field"
-                ng-options="item.name as item.name for (key, item) in metadata[serie.entity].attributes"
+                ng-options="item for item in metadata[serie.entity].attributeKeys"
                 ng-change="setFilterField(filter, serie.entity)"></select>
         </div>
     </div>

diff --git a/frontend/app/views/directives/dashboard/serie.filters.html b/frontend/app/views/directives/dashboard/serie.filters.html
@@ -10,7 +10,7 @@
                 </button>
             </span>
             <select class="form-control" ng-model="filter.field"
-                ng-options="item.name as item.name for (key, item) in metadata[component.options.entity].attributes"
+                ng-options="item for item in metadata[component.options.entity].attributeKeys"
                 ng-change="setFilterField(filter, component.options.entity)"></select>
         </div>
     </div>

diff --git a/frontend/app/views/directives/dashboard/text/serie.filters.html b/frontend/app/views/directives/dashboard/text/serie.filters.html
@@ -10,7 +10,7 @@
                 </button>
             </span>
             <select class="form-control" ng-model="filter.field"
-                ng-options="item.name as item.name for (key, item) in metadata[serie.entity].attributes"
+                ng-options="item for item in metadata[serie.entity].attributeKeys"
                 ng-change="setFilterField(filter, serie.entity)"></select>
         </div>
     </div>

diff --git a/frontend/app/views/partials/case/case.close.html b/frontend/app/views/partials/case/case.close.html
@@ -6,7 +6,9 @@ <h3 class="modal-title">Close Case #{{caze.number}}</h3>
         <div ng-show="!tasksValid">
             <div align="center" class="alert alert-danger">
                 <i class="glyphicon glyphicon-exclamation-sign"></i>
-                This case contains the following open or unassigned tasks
+                This case contains the following open or unassigned tasks. Closing the case will permanently remove the unassigned ones.
+                <br>
+                This action cannot be undone.
             </div>
             <table class="table table-hover">
                 <thead>

diff --git a/frontend/bower.json b/frontend/bower.json
@@ -1,6 +1,6 @@
 {
   "name": "thehive",
-  "version": "4.0.4-1",
+  "version": "4.0.5-1",
   "license": "AGPL-3.0",
   "dependencies": {
     "jquery": "^3.4.1",

diff --git a/frontend/package.json b/frontend/package.json
@@ -1,6 +1,6 @@
 {
     "name": "thehive",
-    "version": "4.0.4-1",
+    "version": "4.0.5-1",
     "license": "AGPL-3.0",
     "repository": {
         "type": "git",

diff --git a/misp/client/src/main/scala/org/thp/misp/dto/Organisation.scala b/misp/client/src/main/scala/org/thp/misp/dto/Organisation.scala
@@ -4,7 +4,7 @@ import java.util.UUID
 
 import play.api.libs.json.{Json, Reads}
 
-case class Organisation(id: String, name: String, description: String, uuid: UUID)
+case class Organisation(id: String, name: String, description: Option[String], uuid: UUID)
 
 object Organisation {
   implicit val reads: Reads[Organisation] = Json.reads[Organisation]

diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispActor.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispActor.scala
@@ -12,8 +12,7 @@ case object Synchro extends MispMessage
 
 class MispActor @Inject() (
     connector: Connector,
-    mispImportSrv: MispImportSrv,
-    userSrv: UserSrv
+    mispImportSrv: MispImportSrv
 ) extends Actor {
   import context.dispatcher
 
@@ -34,7 +33,7 @@ class MispActor @Inject() (
       scheduledSynchronisation.cancel()
       logger.info(s"Synchronising MISP events for ${connector.clients.map(_.name).mkString(",")}")
       connector.clients.filter(_.canImport).foreach { mispClient =>
-        mispImportSrv.syncMispEvents(mispClient)(userSrv.getSystemAuthContext)
+        mispImportSrv.syncMispEvents(mispClient)
       }
       logger.info("MISP synchronisation is complete")
       context.become(receive(context.system.scheduler.scheduleOnce(connector.syncInterval, self, Synchro)))