Skip to content

Commit

Permalink
#312 Add default dashboards during database migration
Browse files Browse the repository at this point in the history
  • Loading branch information
To-om committed Nov 30, 2017
1 parent a68e4e7 commit c951a38
Show file tree
Hide file tree
Showing 6 changed files with 303 additions and 11 deletions.
6 changes: 3 additions & 3 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ mappings in Universal ~= {
file("package/thehive.service") -> "package/thehive.service",
file("package/thehive.conf") -> "package/thehive.conf",
file("package/thehive") -> "package/thehive",
file("package/logback.xml") -> "conf/logback.xml"
)
file("package/logback.xml") -> "conf/logback.xml",
) ++ (file("migration").**(AllPassFilter) pair Path.rebase(file("migration"), "migration"))
}

// Package //
Expand Down Expand Up @@ -115,7 +115,7 @@ packageBin := {
}
// DEB //
linuxPackageMappings in Debian += packageMapping(file("LICENSE") -> "/usr/share/doc/thehive/copyright").withPerms("644")
version in Debian := version.value + "-1"
version in Debian := version.value + "-0"
debianPackageRecommends := Seq("elasticsearch")
debianPackageDependencies += "openjdk-8-jre-headless"
maintainerScripts in Debian := maintainerScriptsFromDirectory(
Expand Down
1 change: 1 addition & 0 deletions migration/12/dashboards/Alert_statistics.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"title":"Alert statistics","definition":{"period":"last3Months","items":[{"type":"container","items":[{"type":"donut","options":{"title":"Alerts by status","entity":"alert","field":"status","query":{},"names":{"New":"New","Updated":"Updated","Ignored":"Ignored","Imported":"Imported"}},"id":"cd063f98-21cc-405c-18a9-af669acae104"},{"type":"donut","options":{"title":"Waiting alerts by type","entity":"alert","field":"type","filters":[{"field":"status","type":"enumeration","value":{"list":[{"text":"New","label":"New"},{"text":"Updated","label":"Updated"}]}}],"query":{"_or":[{"_field":"status","_value":"New"},{"_field":"status","_value":"Updated"}]},"names":{}},"id":"8ca4226f-374e-5315-71b8-5d6a4141d886"},{"type":"donut","options":{"title":"Waiting alerts by source","entity":"alert","field":"source","filters":[{"field":"status","type":"enumeration","value":{"list":[{"text":"New","label":"New"},{"text":"Updated","label":"Updated"}]}}],"query":{"_or":[{"_field":"status","_value":"New"},{"_field":"status","_value":"Updated"}]},"names":{}},"id":"73a986bb-7f53-fc62-6cc8-1e099fadc4b4"}]},{"type":"container","items":[{"type":"bar","options":{"entity":"alert","dateField":"createdAt","interval":"1w","field":"type","stacked":true,"title":"Alert type history","query":{},"names":{}},"id":"62633389-0aa0-827b-ef48-e5bedf7d5e7d"},{"type":"bar","options":{"title":"Alert source history","entity":"alert","dateField":"createdAt","interval":"1w","field":"source","stacked":true,"query":{},"names":{}},"id":"a513f977-e743-9862-0755-9831e9bf080a"}]}],"customPeriod":{"fromDate":null,"toDate":null}},"description":"Alert statistics","status":"Shared"}
244 changes: 244 additions & 0 deletions migration/12/dashboards/Case_statistics.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
{
"description": "case",
"title": "Case statistics",
"definition": {
"period": "last3Months",
"items": [{
"type": "container",
"items": [{
"type": "donut",
"options": {
"title": "Owner of open cases",
"entity": "case",
"field": "owner",
"filters": [{
"field": "status",
"type": "enumeration",
"value": {
"list": [{
"text": "Open",
"label": "Open"
}]
}
}],
"query": {
"_field": "status",
"_value": "Open"
},
"names": {}
},
"id": "4cb4f7d3-eb21-dd61-2a6f-85cf096a2a6e"
}, {
"type": "donut",
"options": {
"title": "Cases by status",
"entity": "case",
"field": "status",
"filters": [],
"names": {
"NoImpact": "NoImpact",
"WithImpact": "WithImpact",
"NotApplicable": "NotApplicable",
"Open": "Open",
"Resolved": "Resolved",
"Deleted": "Deleted"
},
"query": {}
},
"id": "84b81a65-4b3c-2b26-421e-fd7453d92f3e"
}]
}, {
"type": "container",
"items": [{
"type": "donut",
"options": {
"title": "Revolved cases by resolution",
"entity": "case",
"field": "resolutionStatus",
"filters": [{
"field": "status",
"type": "enumeration",
"value": {
"list": [{
"text": "Resolved",
"label": "Resolved"
}]
}
}],
"query": {
"_field": "status",
"_value": "Resolved"
},
"names": {
"FalsePositive": "FalsePositive",
"Duplicated": "Duplicated",
"Indeterminate": "Indeterminate",
"TruePositive": "TruePositive",
"Other": "Other"
}
},
"id": "ede6e87a-2e39-5556-b421-1c4cd73a74b1"
}, {
"type": "donut",
"options": {
"title": "Case tags",
"entity": "case",
"field": "tags",
"query": {},
"names": {}
},
"id": "a9e47a5d-3c84-4949-b941-a60ea3c41e81"
}]
}, {
"type": "container",
"items": [{
"type": "bar",
"options": {
"entity": "case",
"dateField": "createdAt",
"interval": "1w",
"field": "owner",
"stacked": true,
"query": {},
"names": {},
"title": "Case owner history"
},
"id": "b5bb88c6-0a76-ca85-c4b6-5096199ddf80"
}, {
"type": "bar",
"options": {
"entity": "case",
"dateField": "createdAt",
"interval": "1w",
"field": "severity",
"stacked": true,
"query": {},
"names": {
"1": "low",
"2": "medium",
"3": "high"
},
"title": "Case severity history"
},
"id": "9bdac0ad-441b-2be3-9e6e-342968be5315"
}, {
"type": "bar",
"options": {
"entity": "case",
"dateField": "createdAt",
"interval": "1w",
"field": "tlp",
"stacked": true,
"title": "Case TLP history",
"query": {},
"names": {
"0": "white",
"1": "green",
"2": "amber",
"3": "red"
}
},
"id": "72157fd6-efb4-cf0c-a281-7eacc3c32a4f"
}]
}, {
"type": "container",
"items": [{
"type": "line",
"options": {
"title": "Case over time",
"entity": "case",
"field": "createdAt",
"interval": "1w",
"series": [{
"agg": "avg",
"field": "computed.handlingDurationInHours",
"type": "line",
"filters": [{
"field": "status",
"type": "enumeration",
"value": {
"list": [{
"text": "Resolved",
"label": "Resolved"
}]
}
}],
"query": {
"_field": "status",
"_value": "Resolved"
}
}, {
"agg": "count",
"field": null,
"type": "bar"
}],
"query": {}
},
"id": "377784a7-49c2-50aa-2eba-acc862a0b841"
}]
}, {
"type": "container",
"items": [{
"type": "donut",
"options": {
"title": "TLP of open cases",
"entity": "case",
"field": "tlp",
"filters": [{
"field": "status",
"type": "enumeration",
"value": {
"list": [{
"text": "Open",
"label": "Open"
}]
}
}],
"query": {
"_field": "status",
"_value": "Open"
},
"names": {
"0": "white",
"1": "green",
"2": "amber",
"3": "red"
}
},
"id": "4c7bb013-c87f-7f17-0892-e20af2a0dcac"
}, {
"type": "donut",
"options": {
"title": "Severity of open cases",
"entity": "case",
"field": "severity",
"filters": [{
"field": "status",
"type": "enumeration",
"value": {
"list": [{
"text": "Open",
"label": "Open"
}]
}
}],
"query": {
"_field": "status",
"_value": "Open"
},
"names": {
"1": "low",
"2": "medium",
"3": "high"
}
},
"id": "d943c6f4-61d8-b4dd-7a3a-56067829727a"
}]
}],
"customPeriod": {
"fromDate": null,
"toDate": null
}
},
"status": "Shared"
}
1 change: 1 addition & 0 deletions migration/12/dashboards/Job_statistics.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"definition":{"period":"last3Months","items":[{"type":"container","items":[{"type":"donut","options":{"title":"Top analyzers","entity":"case_artifact_job","field":"analyzerId","query":{},"names":{}},"id":"1eaa4dfa-5b14-50b6-e442-8729363f6f66"},{"type":"donut","options":{"title":"Cortex instance use","entity":"case_artifact_job","field":"cortexId","query":{},"names":{}},"id":"c501c2d3-9779-1d2a-6d85-bb2bd68260f5"}]},{"type":"container","items":[{"type":"bar","options":{"title":"Job owners","entity":"case_artifact_job","dateField":"createdAt","interval":"1w","field":"createdBy","stacked":true,"query":{},"names":{}},"id":"bc10b554-aa4c-6fce-c4bb-b906b9b0e398"},{"type":"bar","options":{"title":"Analyzer history","entity":"case_artifact_job","dateField":"createdAt","interval":"1w","field":"analyzerId","stacked":true,"query":{},"names":{}},"id":"cd6d0dc1-a77d-be9d-e7dd-c6a8c79b0898"}]}],"customPeriod":{"fromDate":null,"toDate":null}},"title":"Job statistics","status":"Shared","description":"Job statistics"}
1 change: 1 addition & 0 deletions migration/12/dashboards/Observable_statistics .json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"definition":{"period":"last3Months","items":[{"type":"container","items":[{"type":"donut","options":{"title":"Observables by type","entity":"case_artifact","field":"dataType","query":{},"names":{"fqdn":"fqdn","url":"url","regexp":"regexp","mail":"mail","hash":"hash","registry":"registry","uri_path":"uri_path","truc":"truc","ip":"ip","user-agent":"user-agent","autonomous-system":"autonomous-system","file":"file","mail_subject":"mail_subject","filename":"filename","other":"other","domain":"domain"}},"id":"6ee86a99-3f40-1960-fd4d-398a1da5b76e"},{"type":"donut","options":{"title":"Observables by attachment content type","entity":"case_artifact","field":"attachment.contentType","query":{"_field":"dataType","_value":"file"},"names":{},"filters":[{"field":"dataType","type":"enumeration","value":{"list":[{"text":"file","label":"file"}]}}]},"id":"b6110238-3074-4e85-674f-4bc56829e68a"}]},{"type":"container","items":[{"type":"donut","options":{"title":"Observable tags","entity":"case_artifact","field":"tags","query":{},"names":{}},"id":"70bbc0a5-1692-4e46-ebac-8769952ad9c0"},{"type":"donut","options":{"title":"Observables by TLP","entity":"case_artifact","field":"tlp","query":{},"names":{"0":"white","1":"green","2":"amber","3":"red"},"colors":{"0":"#bdf0ea","1":"#48e80f","2":"#e0a91a","3":"#f02626"}},"id":"633fbe97-805e-6123-3330-29f5c8f45f13"}]},{"type":"container","items":[{"type":"donut","options":{"title":"Observables by IOC flag","entity":"case_artifact","field":"ioc","query":{},"names":{}},"id":"771a3bdf-e437-ac3a-384d-23be91a25b07"},{"type":"line","options":{"title":"Observables over time","entity":"case_artifact","field":"createdAt","interval":"1w","series":[{"agg":"count","field":null,"type":"area-spline","filters":[{"field":"ioc","type":"boolean","value":true}],"label":"IOC","query":{"_field":"ioc","_value":true}},{"agg":"count","field":null,"type":"area-spline","label":"non-IOC","filters":[{"field":"ioc","type":"boolean","value":false}],"query":{"_field":"ioc","_value":false}}],"stacked":true,"query":{}},"id":"e5ed24a6-51ed-ecc4-9db0-ce837fd84214"}]}],"customPeriod":{"fromDate":null,"toDate":null}},"status":"Shared","title":"Observable statistics","description":"Observable statistics"}
61 changes: 53 additions & 8 deletions thehive-backend/app/models/Migration.scala
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package models

import java.nio.file.{ Files, Path, Paths }
import java.util.Date
import javax.inject.{ Inject, Singleton }

import scala.collection.JavaConverters._
import scala.concurrent.{ ExecutionContext, Future }
import scala.math.BigDecimal.int2bigDecimal
import scala.util.Try
Expand All @@ -14,8 +16,9 @@ import play.api.{ Configuration, Logger }
import akka.NotUsed
import akka.stream.Materializer
import akka.stream.scaladsl.Source
import services.AlertSrv
import services.{ AlertSrv, DashboardSrv }

import org.elastic4play.controllers.Fields
import org.elastic4play.services.JsonFormat.attachmentFormat
import org.elastic4play.services._
import org.elastic4play.utils.Hasher
Expand All @@ -30,12 +33,16 @@ class Migration(
datastoreName: String,
dblists: DBLists,
eventSrv: EventSrv,
dashboardSrv: DashboardSrv,
userSrv: UserSrv,
implicit val ec: ExecutionContext,
implicit val materializer: Materializer) extends MigrationOperations {
@Inject() def this(
configuration: Configuration,
dblists: DBLists,
eventSrv: EventSrv,
dashboardSrv: DashboardSrv,
userSrv: UserSrv,
ec: ExecutionContext,
materializer: Materializer) = {
this(
Expand All @@ -44,7 +51,10 @@ class Migration(
configuration.get[Seq[String]]("datastore.hash.extra"),
configuration.get[String]("datastore.name"),
dblists,
eventSrv, ec, materializer)
eventSrv,
dashboardSrv,
userSrv,
ec, materializer)
}

import org.elastic4play.services.Operation._
Expand All @@ -54,18 +64,53 @@ class Migration(

override def beginMigration(version: Int): Future[Unit] = Future.successful(())

private def readJsonFile(file: Path): JsValue = {
val source = scala.io.Source.fromFile(file.toFile)
try Json.parse(source.mkString)
finally source.close()
}

private def addDataTypes(dataTypes: Seq[String]): Future[Unit] = {
val dataTypeList = dblists.apply("list_artifactDataType")
Future
.traverse(dataTypes) { dt
dataTypeList.addItem(dt)
.map(_ ())
.recover {
case error logger.error(s"Failed to add dataType $dt during migration", error)
}
}
.map(_ ())
}

private def addDashboards(version: Int): Future[Unit] = {
userSrv.inInitAuthContext { implicit authContext
val dashboardsPath = Paths.get("migration").resolve(version.toString).resolve("dashboards")
val dashboards = for {
dashboardFile Try(Files.newDirectoryStream(dashboardsPath, "*.json").asScala).getOrElse(Nil)
if Files.isReadable(dashboardFile)
dashboardJson Try(readJsonFile(dashboardFile).as[JsObject]).toOption
dashboardDefinition = (dashboardJson \ "definition").as[JsValue].toString
dash = dashboardSrv.create(Fields(dashboardJson + ("definition" -> JsString(dashboardDefinition))))
.map(_ ())
.recover {
case error logger.error(s"Failed to create dashboard $dashboardFile during migration", error)
}
} yield dash
Future.sequence(dashboards).map(_ ())
}
}

override def endMigration(version: Int): Future[Unit] = {
if (requireUpdateMispAlertArtifact) {
logger.info("Retrieve MISP attribute to update alerts")
eventSrv.publish(UpdateMispAlertArtifact())
}
logger.info("Updating observable data type list")
val dataTypes = dblists.apply("list_artifactDataType")
Future.sequence(Seq("filename", "fqdn", "url", "user-agent", "domain", "ip", "mail_subject", "hash", "mail",
"registry", "uri_path", "regexp", "other", "file", "autonomous-system")
.map(dt dataTypes.addItem(dt).recover { case _ () }))
.map(_ ())
.recover { case _ () }
addDataTypes(Seq("filename", "fqdn", "url", "user-agent", "domain", "ip", "mail_subject", "hash", "mail",
"registry", "uri_path", "regexp", "other", "file", "autonomous-system"))
.andThen { case _ addDashboards(version + 1) }

}

override val operations: PartialFunction[DatabaseState, Seq[Operation]] = {
Expand Down

0 comments on commit c951a38

Please sign in to comment.