Merge branch 'hotfix/4.0.3' into master-th4

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Change Log
 
+## [4.0.3](https://github.com/TheHive-Project/TheHive/milestone/66) (2020-12-22)
+
+**Implemented enhancements:**
+
+- Providing output details for Responders [\#1293](https://github.com/TheHive-Project/TheHive/issues/1293)
+- [Enhancement] Change artifacts by observables on the onMouseOver tooltip of the eye icon of observable [\#1695](https://github.com/TheHive-Project/TheHive/issues/1695)
+- [Bug] Enhance support of S3 for attachment storage [\#1705](https://github.com/TheHive-Project/TheHive/issues/1705)
+- Update the headers of basic info sections [\#1710](https://github.com/TheHive-Project/TheHive/issues/1710)
+- [Enhancement] Add poll duration config for UI Stream [\#1720](https://github.com/TheHive-Project/TheHive/issues/1720)
+
+**Fixed bugs:**
+
+- [Bug] MISP filters are not correctly implemented [\#1685](https://github.com/TheHive-Project/TheHive/issues/1685)
+- [Bug] The query "getObservable" doesn't work for alert observables [\#1691](https://github.com/TheHive-Project/TheHive/issues/1691)
+- Click analyzers mini-report does not load the full report [\#1694](https://github.com/TheHive-Project/TheHive/issues/1694)
+- [TH4] Import file observable in gui generate error [\#1697](https://github.com/TheHive-Project/TheHive/issues/1697)
+- [Bug] Cannot search for alerts per observables [\#1707](https://github.com/TheHive-Project/TheHive/issues/1707)
+- [Bug] Serialization problem in cluster mode [\#1708](https://github.com/TheHive-Project/TheHive/issues/1708)
+- [Bug] Issue with sorting [\#1716](https://github.com/TheHive-Project/TheHive/issues/1716)
+- [Bug] Identical URL Observables can be added multiple times to the same case [\#1718](https://github.com/TheHive-Project/TheHive/issues/1718)
+
 ## [4.0.2](https://github.com/TheHive-Project/TheHive/milestone/64) (2020-11-20)
 
 **Implemented enhancements:**
@@ -218,7 +239,6 @@
 - A (received) Shared Case is displayed as sender/owner [\#1245](https://github.com/TheHive-Project/TheHive/issues/1245)
 - FR: Alignment of case custom-fields (metrics) [\#1246](https://github.com/TheHive-Project/TheHive/issues/1246)
 - Add information about the age of a Case  [\#1257](https://github.com/TheHive-Project/TheHive/issues/1257)
-- Providing output details for Responders [\#1293](https://github.com/TheHive-Project/TheHive/issues/1293)
 - Add support to multi-factor authentication [\#1303](https://github.com/TheHive-Project/TheHive/issues/1303)
 - Add support to webhooks [\#1306](https://github.com/TheHive-Project/TheHive/issues/1306)
 

build.sbt

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.0.2-1"
+val thehiveVersion         = "4.0.3-1"
 val scala212               = "2.12.12"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)

cortex/connector/src/main/resources/play/reference-overrides.conf

@@ -0,0 +1,13 @@
+akka {
+  actor {
+    serializers {
+      cortex-schema-updater = "org.thp.thehive.connector.cortex.models.SchemaUpdaterSerializer"
+      cortex-jobs = "org.thp.thehive.connector.cortex.services.CortexSerializer"
+    }
+
+    serialization-bindings {
+      "org.thp.thehive.connector.cortex.models.SchemaUpdaterMessage" = cortex-schema-updater
+      "org.thp.thehive.connector.cortex.services.CortexActorMessage" = cortex-jobs
+    }
+  }
+}

cortex/connector/src/main/resources/reference.conf

@@ -22,4 +22,4 @@ cortex = {
   //    # HTTP client configuration (SSL and proxy)
   //    #  ws {}
   //  }]
-}
+}

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/JobCtrl.scala

@@ -1,6 +1,7 @@
 package org.thp.thehive.connector.cortex.controllers.v0
 
 import com.google.inject.name.Named
+
 import javax.inject.{Inject, Singleton}
 import org.thp.scalligraph.controllers.{Entrypoint, FieldsParser}
 import org.thp.scalligraph.models.{Database, UMapping}
@@ -14,7 +15,7 @@ import org.thp.thehive.connector.cortex.services.JobOps._
 import org.thp.thehive.connector.cortex.services.JobSrv
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.controllers.v0.{OutputParam, PublicData, QueryCtrl}
-import org.thp.thehive.models.{Permissions, RichCase, RichObservable}
+import org.thp.thehive.models.{Observable, Permissions, RichCase, RichObservable}
 import org.thp.thehive.services.ObservableOps._
 import org.thp.thehive.services.ObservableSrv
 import play.api.mvc.{Action, AnyContent, Results}
@@ -93,6 +94,9 @@ class PublicJob @Inject() (jobSrv: JobSrv) extends PublicData with JobRenderer {
       }
     )
   override val outputQuery: Query = Query.outputWithContext[RichJob, Traversal.V[Job]]((jobSteps, authContext) => jobSteps.richJob(authContext))
+  override val extraQueries: Seq[ParamQuery[_]] = Seq(
+    Query[Traversal.V[Observable], Traversal.V[Job]]("jobs", (jobTraversal, _) => jobTraversal.jobs)
+  )
   override val publicProperties: PublicProperties = PublicPropertyListBuilder[Job]
     .property("analyzerId", UMapping.string)(_.rename("workerId").readonly)
     .property("cortexId", UMapping.string.optional)(_.field.readonly)

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Job.scala

@@ -1,12 +1,12 @@
 package org.thp.thehive.connector.cortex.models
 
-import java.util.Date
-
 import org.thp.scalligraph.models.Entity
 import org.thp.scalligraph.{BuildEdgeEntity, BuildVertexEntity, EntityId}
 import org.thp.thehive.models.{Observable, RichObservable}
 import play.api.libs.json.{Format, JsObject, Json}
 
+import java.util.Date
+
 object JobStatus extends Enumeration {
   val InProgress, Success, Failure, Waiting, Deleted = Value
 

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterActor.scala

@@ -4,22 +4,20 @@ import akka.actor.{Actor, ActorRef, ActorSystem, PoisonPill, Props}
 import akka.cluster.singleton.{ClusterSingletonManager, ClusterSingletonManagerSettings, ClusterSingletonProxy, ClusterSingletonProxySettings}
 import akka.pattern.ask
 import akka.util.Timeout
-import javax.inject.{Inject, Named, Provider, Singleton}
 import org.thp.scalligraph.models.Database
 import org.thp.thehive.services.LocalUserSrv
 import play.api.Logger
 
+import javax.inject.{Inject, Named, Provider, Singleton}
 import scala.concurrent.Await
 import scala.concurrent.duration.DurationInt
-import scala.util.Try
 
 @Singleton
 class DatabaseProvider @Inject() (
     cortexSchema: CortexSchemaDefinition,
     @Named("with-thehive-schema") database: Database,
     actorSystem: ActorSystem
 ) extends Provider[Database] {
-  import SchemaUpdaterActor._
   lazy val schemaUpdaterActor: ActorRef = {
     val singletonManager =
       actorSystem.actorOf(
@@ -42,43 +40,37 @@ class DatabaseProvider @Inject() (
 
   override def get(): Database = {
     implicit val timeout: Timeout = Timeout(5.minutes)
-    Await.result(schemaUpdaterActor ? RequestDBStatus, timeout.duration) match {
-      case DBStatus(status) =>
-        status.get
-        database
+    Await.result(schemaUpdaterActor ? RequestDB, timeout.duration) match {
+      case DBReady => database
     }
   }
 }
 
-object SchemaUpdaterActor {
-  case object RequestDBStatus
-  case class DBStatus(status: Try[Unit])
-}
+sealed trait SchemaUpdaterMessage
+case object RequestDB extends SchemaUpdaterMessage
+case object DBReady   extends SchemaUpdaterMessage
 
 class SchemaUpdaterActor @Inject() (cortexSchema: CortexSchemaDefinition, database: Database) extends Actor {
-  import SchemaUpdaterActor._
   lazy val logger: Logger = Logger(getClass)
 
-  def update(): Try[Unit] =
+  def update(): Unit = {
     cortexSchema
       .update(database)(LocalUserSrv.getSystemAuthContext)
       .recover {
         case error => logger.error(s"Database with CortexSchema schema update failure", error)
       }
+    ()
+  }
 
   override def receive: Receive = {
-    case RequestDBStatus =>
-      val status = update()
-      sender ! DBStatus(status)
-      context.become(receive(status))
+    case RequestDB =>
+      update()
+      sender ! DBReady
+      context.become(databaseUpToDate)
   }
 
-  def receive(status: Try[Unit]): Receive = {
-    case RequestDBStatus =>
-      status.fold({ _ =>
-        val newStatus = update()
-        sender ! DBStatus(newStatus)
-        context.become(receive(newStatus))
-      }, _ => sender ! DBStatus(status))
+  def databaseUpToDate: Receive = {
+    case RequestDB =>
+      sender ! DBReady
   }
 }

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterSerializer.scala

@@ -0,0 +1,25 @@
+package org.thp.thehive.connector.cortex.models
+
+import akka.serialization.Serializer
+
+import java.io.NotSerializableException
+
+class SchemaUpdaterSerializer extends Serializer {
+  override def identifier: Int = -639734235
+
+  override def includeManifest: Boolean = false
+
+  override def toBinary(o: AnyRef): Array[Byte] =
+    o match {
+      case RequestDB => Array(0)
+      case DBReady   => Array(1)
+      case _         => throw new NotSerializableException
+    }
+
+  override def fromBinary(bytes: Array[Byte], manifest: Option[Class[_]]): AnyRef =
+    bytes(0) match {
+      case 0 => RequestDB
+      case 1 => DBReady
+      case _ => throw new NotSerializableException
+    }
+}

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala

@@ -90,7 +90,7 @@ class ActionOperationSrv @Inject() (
       case AddLogToTask(content, _) =>
         for {
           t <- relatedTask.fold[Try[Task with Entity]](Failure(InternalError("Unable to apply action AddLogToTask without task")))(Success(_))
-          _ <- logSrv.create(Log(content, new Date(), deleted = false), t)
+          _ <- logSrv.create(Log(content, new Date(), deleted = false), t, None)
         } yield updateOperation(operation)
 
       case AddArtifactToCase(_, dataType, dataMessage) =>

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala

@@ -18,7 +18,6 @@ import org.thp.thehive.connector.cortex.controllers.v0.Conversion._
 import org.thp.thehive.connector.cortex.models._
 import org.thp.thehive.connector.cortex.services.ActionOps._
 import org.thp.thehive.connector.cortex.services.Conversion._
-import org.thp.thehive.connector.cortex.services.CortexActor.CheckJob
 import org.thp.thehive.controllers.v0.Conversion._
 import org.thp.thehive.models._
 import org.thp.thehive.services.AlertOps._

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/CortexActor.scala

@@ -1,39 +1,37 @@
 package org.thp.thehive.connector.cortex.services
 
-import java.util.Date
-
 import akka.actor._
 import akka.pattern.pipe
-import javax.inject.Inject
 import org.thp.client.ApplicationError
-import org.thp.cortex.dto.v0.{JobStatus, JobType, OutputJob => CortexJob}
+import org.thp.cortex.dto.v0.{JobStatus, JobType, OutputJob}
 import org.thp.scalligraph.EntityId
 import org.thp.scalligraph.auth.AuthContext
 import play.api.Logger
 
+import java.util.Date
+import javax.inject.Inject
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 
-object CortexActor {
-  final case class CheckJob(
-      jobId: Option[EntityId],
-      cortexJobId: String,
-      actionId: Option[EntityId],
-      cortexId: String,
-      authContext: AuthContext
-  )
-
-  final private case object CheckJobs
-  final private case object CheckJobsKey
-  final private case object FirstCheckJobs
-}
-
+sealed trait CortexActorMessage
+case class RemoteJob(job: OutputJob) extends CortexActorMessage
+case class CheckJob(
+    jobId: Option[EntityId],
+    cortexJobId: String,
+    actionId: Option[EntityId],
+    cortexId: String,
+    authContext: AuthContext
+) extends CortexActorMessage
+
+private case object CheckJobs extends CortexActorMessage
+private case object CheckJobsKey
+private case object FirstCheckJobs extends CortexActorMessage
+// FIXME Add serializer
 /**
   * This actor is primarily used to check Job statuses on regular
   * ticks using the provided client for each job
   */
 class CortexActor @Inject() (connector: Connector, jobSrv: JobSrv, actionSrv: ActionSrv) extends Actor with Timers {
-  import CortexActor._
   implicit val ec: ExecutionContext = context.dispatcher
   lazy val logger: Logger           = Logger(getClass)
 
@@ -66,35 +64,36 @@ class CortexActor @Inject() (connector: Connector, jobSrv: JobSrv, actionSrv: Ac
                   .getReport(cortexJobId, 1.second)
                   .recover { // this is a workaround for a timeout bug in Cortex
                     case ApplicationError(500, body) if (body \ "type").asOpt[String].contains("akka.pattern.AskTimeoutException") =>
-                      CortexJob(cortexJobId, "", "", "", new Date, None, None, JobStatus.InProgress, None, None, "", "", None, JobType.analyzer)
+                      OutputJob(cortexJobId, "", "", "", new Date, None, None, JobStatus.InProgress, None, None, "", "", None, JobType.analyzer)
                   }
+                  .map(RemoteJob)
                   .pipeTo(self)
                 ()
               }
         }
 
-    case cortexJob: CortexJob if cortexJob.status == JobStatus.Success || cortexJob.status == JobStatus.Failure =>
-      checkedJobs.find(_.cortexJobId == cortexJob.id) match {
-        case Some(CheckJob(Some(jobId), cortexJobId, _, cortexId, authContext)) if cortexJob.`type` == JobType.analyzer =>
-          logger.info(s"Job $cortexJobId in cortex $cortexId has finished with status ${cortexJob.status}, updating job $jobId")
-          jobSrv.finished(cortexId, jobId, cortexJob)(authContext)
-          context.become(receive(checkedJobs.filterNot(_.cortexJobId == cortexJob.id), failuresCount))
+    case RemoteJob(job) if job.status == JobStatus.Success || job.status == JobStatus.Failure =>
+      checkedJobs.find(_.cortexJobId == job.id) match {
+        case Some(CheckJob(Some(jobId), cortexJobId, _, cortexId, authContext)) if job.`type` == JobType.analyzer =>
+          logger.info(s"Job $cortexJobId in cortex $cortexId has finished with status ${job.status}, updating job $jobId")
+          jobSrv.finished(cortexId, jobId, job)(authContext)
+          context.become(receive(checkedJobs.filterNot(_.cortexJobId == job.id), failuresCount))
 
-        case Some(CheckJob(_, cortexJobId, Some(actionId), cortexId, authContext)) if cortexJob.`type` == JobType.responder =>
-          logger.info(s"Job $cortexJobId in cortex $cortexId has finished with status ${cortexJob.status}, updating action $actionId")
-          actionSrv.finished(actionId, cortexJob)(authContext)
-          context.become(receive(checkedJobs.filterNot(_.cortexJobId == cortexJob.id), failuresCount))
+        case Some(CheckJob(_, cortexJobId, Some(actionId), cortexId, authContext)) if job.`type` == JobType.responder =>
+          logger.info(s"Job $cortexJobId in cortex $cortexId has finished with status ${job.status}, updating action $actionId")
+          actionSrv.finished(actionId, job)(authContext)
+          context.become(receive(checkedJobs.filterNot(_.cortexJobId == job.id), failuresCount))
 
         case Some(_) =>
-          logger.error(s"CortexActor received job output $cortexJob but with unknown type ${cortexJob.`type`}")
+          logger.error(s"CortexActor received job output $job but with unknown type ${job.`type`}")
 
         case None =>
-          logger.error(s"CortexActor received job output $cortexJob but did not have it in state $checkedJobs")
+          logger.error(s"CortexActor received job output $job but did not have it in state $checkedJobs")
       }
-    case cortexJob: CortexJob if cortexJob.status == JobStatus.InProgress || cortexJob.status == JobStatus.Waiting =>
-      logger.info(s"CortexActor received ${cortexJob.status} from client, retrying in ${connector.refreshDelay}")
+    case RemoteJob(job) if job.status == JobStatus.InProgress || job.status == JobStatus.Waiting =>
+      logger.info(s"CortexActor received ${job.status} from client, retrying in ${connector.refreshDelay}")
 
-    case _: CortexJob =>
+    case _: RemoteJob =>
       logger.warn(s"CortexActor received JobStatus.Unknown from client, retrying in ${connector.refreshDelay}")
 
     case Status.Failure(e) if failuresCount < connector.maxRetryOnError =>