From db653f84a7057eb595eebc27523ed716509eb594 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 14:54:58 +0100 Subject: [PATCH 01/38] Adapt message structure --- .../osmogrid/guardian/OsmoGridGuardian.scala | 202 ++++++++++-------- .../osmogrid/io/input/InputDataProvider.scala | 14 +- .../osmogrid/io/output/ResultListener.scala | 11 +- .../edu/ie3/osmogrid/lv/LvCoordinator.scala | 91 ++++---- .../edu/ie3/osmogrid/lv/LvGenerator.scala | 6 +- .../ie3/osmogrid/lv/LvRegionCoordinator.scala | 16 +- .../osmogrid/main/RunOsmoGridStandalone.scala | 4 +- 7 files changed, 177 insertions(+), 167 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 30657f75..7052127c 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -17,9 +17,8 @@ import edu.ie3.datamodel.utils.ContainerUtils import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.cfg.OsmoGridConfig.Generation import edu.ie3.osmogrid.io.input.InputDataProvider -import edu.ie3.osmogrid.io.input.InputDataProvider.InputDataEvent import edu.ie3.osmogrid.io.output.ResultListener -import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, ResultEvent} +import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, Request} import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids @@ -28,110 +27,123 @@ import scala.util.{Failure, Success, Try} object OsmoGridGuardian { - sealed trait OsmoGridGuardianEvent - final case class Run(cfg: OsmoGridConfig) extends OsmoGridGuardianEvent - object InputDataProviderDied extends OsmoGridGuardianEvent - object ResultEventListenerDied extends OsmoGridGuardianEvent - object LvCoordinatorDied extends OsmoGridGuardianEvent - final case class RepLvGrids(grids: Vector[SubGridContainer]) - extends OsmoGridGuardianEvent + /* Messages, that are understood and sent */ + sealed trait Request + final case class Run(cfg: OsmoGridConfig) extends Request + object InputDataProviderDied extends Request + object ResultEventListenerDied extends Request + object LvCoordinatorDied extends Request - def apply(): Behavior[OsmoGridGuardianEvent] = idle() + sealed trait Response + final case class RepLvGrids(grids: Seq[SubGridContainer]) extends Response - private def idle(): Behavior[OsmoGridGuardianEvent] = - Behaviors.receive { (ctx, msg) => - msg match { - case Run(cfg) => - ctx.log.info("Initializing grid generation!") + /* Container class for message adapters as well as wrapping classes themselves */ + private final case class MessageAdapters( + lvCoordinator: ActorRef[LvCoordinator.Response] + ) + private object MessageAdapters { + final case class WrappedLvCoordinatorResponse( + response: LvCoordinator.Response + ) extends Request + } + + def apply(): Behavior[Request] = Behaviors.setup { context => + /* Define message adapters */ + val messageAdapters = + MessageAdapters( + context.messageAdapter(msg => + MessageAdapters.WrappedLvCoordinatorResponse(msg) + ) + ) + + idle(messageAdapters) + } + + private def idle(msgAdapters: MessageAdapters): Behavior[Request] = + Behaviors.receive { + case (ctx, Run(cfg)) => + ctx.log.info("Initializing grid generation!") - ctx.log.info("Starting input data provider") - val inputProvider = - ctx.spawn(InputDataProvider(cfg.input), "InputDataProvider") - ctx.watchWith(inputProvider, InputDataProviderDied) - ctx.log.debug("Starting output data listener") - val resultEventListener = - ctx.spawn(ResultListener(cfg.output), "ResultListener") - ctx.watchWith(resultEventListener, ResultEventListenerDied) + ctx.log.info("Starting input data provider") + val inputProvider = + ctx.spawn(InputDataProvider(cfg.input), "InputDataProvider") + ctx.watchWith(inputProvider, InputDataProviderDied) + ctx.log.debug("Starting output data listener") + val resultEventListener = + ctx.spawn(ResultListener(cfg.output), "ResultListener") + ctx.watchWith(resultEventListener, ResultEventListenerDied) - /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ - cfg.generation match { - case Generation(Some(lvConfig)) => - ctx.log.debug("Starting low voltage grid coordinator.") - val lvCoordinator = ctx.spawn(LvCoordinator(), "LvCoordinator") - ctx.watchWith(lvCoordinator, LvCoordinatorDied) - lvCoordinator ! ReqLvGrids(lvConfig, ctx.self) - awaitLvGrids(inputProvider, resultEventListener) - case unsupported => - ctx.log.error( - "Received unsupported grid generation config. Bye, bye." - ) - Behaviors.stopped - } - case InputDataProviderDied => - ctx.log.error("Input data provider died. That's bad...") - Behaviors.stopped - case ResultEventListenerDied => - ctx.log.error("Result event listener died. That's bad...") - Behaviors.stopped - case LvCoordinatorDied => - ctx.log.error("Lv coordinator died. That's bad...") - Behaviors.stopped - case unsupported => - ctx.log.error(s"Received unsupported message '$unsupported'.") - Behaviors.stopped - } + /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ + cfg.generation match { + case Generation(Some(lvConfig)) => + ctx.log.debug("Starting low voltage grid coordinator.") + val lvCoordinator = ctx.spawn(LvCoordinator(), "LvCoordinator") + ctx.watchWith(lvCoordinator, LvCoordinatorDied) + lvCoordinator ! ReqLvGrids(lvConfig, msgAdapters.lvCoordinator) + awaitLvGrids(inputProvider, resultEventListener) + case unsupported => + ctx.log.error( + "Received unsupported grid generation config. Bye, bye." + ) + Behaviors.stopped + } + case (ctx, unsupported) => + ctx.log.error(s"Received unsupported message '$unsupported'.") + Behaviors.stopped } private def awaitLvGrids( - inputDataProvider: ActorRef[InputDataEvent], - resultListener: ActorRef[ResultEvent] - ): Behaviors.Receive[OsmoGridGuardianEvent] = - Behaviors.receive { (ctx, msg) => - msg match { - case RepLvGrids(lvGrids) => - ctx.log.info(s"Received ${lvGrids.length} lv grids. Join them.") - Try(ContainerUtils.combineToJointGrid(lvGrids.asJava)) match { - case Success(jointGrid) => - resultListener ! GridResult(jointGrid, ctx.self) - awaitShutDown(inputDataProvider) - case Failure(exception) => - ctx.log.error( - "Combination of received sub-grids failed. Shutting down." - ) - Behaviors.stopped - } - case unsupported => - ctx.log.error( - s"Received unsupported message while waiting for lv grids. Unsupported: $unsupported" - ) - Behaviors.stopped - } + inputDataProvider: ActorRef[InputDataProvider.Request], + resultListener: ActorRef[ResultListener.Request] + ): Behaviors.Receive[Request] = + Behaviors.receive { + case ( + ctx, + MessageAdapters.WrappedLvCoordinatorResponse( + LvCoordinator.RepLvGrids(lvGrids) + ) + ) => + ctx.log.info(s"Received ${lvGrids.length} lv grids. Join them.") + Try(ContainerUtils.combineToJointGrid(lvGrids.asJava)) match { + case Success(jointGrid) => + resultListener ! GridResult(jointGrid, ctx.self) + awaitShutDown(inputDataProvider) + case Failure(exception) => + ctx.log.error( + "Combination of received sub-grids failed. Shutting down." + ) + Behaviors.stopped + } + case (ctx, unsupported) => + ctx.log.error( + s"Received unsupported message while waiting for lv grids. Unsupported: $unsupported" + ) + Behaviors.stopped } private def awaitShutDown( - inputDataProvider: ActorRef[InputDataEvent], + inputDataProvider: ActorRef[InputDataProvider.Request], resultListenerTerminated: Boolean = false, inputDataProviderTerminated: Boolean = false - ): Behaviors.Receive[OsmoGridGuardianEvent] = Behaviors.receive { - (ctx, msg) => - msg match { - case ResultEventListenerDied => - ctx.log.info("Result listener finished handling the result.") - ctx.log.debug("Shut down input data provider.") - awaitShutDown(inputDataProvider, resultListenerTerminated = true) - case InputDataProviderDied if resultListenerTerminated => - /* That's the fine case */ - ctx.log.info("Input data provider shut down.") - Behaviors.stopped - case InputDataProviderDied => - /* That's the malicious case */ - ctx.log.error( - "Input data provider unexpectedly died during shutdown was initiated." - ) - Behaviors.stopped - case unsupported => - ctx.log.error(s"Received an unsupported message $unsupported.") - Behaviors.stopped - } + ): Behaviors.Receive[Request] = Behaviors.receive { (ctx, msg) => + msg match { + case ResultEventListenerDied => + ctx.log.info("Result listener finished handling the result.") + ctx.log.debug("Shut down input data provider.") + awaitShutDown(inputDataProvider, resultListenerTerminated = true) + case InputDataProviderDied if resultListenerTerminated => + /* That's the fine case */ + ctx.log.info("Input data provider shut down.") + Behaviors.stopped + case InputDataProviderDied => + /* That's the malicious case */ + ctx.log.error( + "Input data provider unexpectedly died during shutdown was initiated." + ) + Behaviors.stopped + case unsupported => + ctx.log.error(s"Received an unsupported message $unsupported.") + Behaviors.stopped + } } } diff --git a/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala b/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala index fafbd873..5cc866e0 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala @@ -10,17 +10,19 @@ import akka.actor.typed.Behavior import akka.actor.typed.ActorRef import akka.actor.typed.scaladsl.Behaviors import edu.ie3.osmogrid.cfg.OsmoGridConfig -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.OsmoGridGuardianEvent +import edu.ie3.osmogrid.guardian.OsmoGridGuardian object InputDataProvider { - sealed trait InputDataEvent + sealed trait Request final case class Read() - extends InputDataEvent // todo this read method should contain configuration parameters for the actual source + potential filter options - final case class Terminate(replyTo: ActorRef[OsmoGridGuardianEvent]) - extends InputDataEvent + extends Request // todo this read method should contain configuration parameters for the actual source + potential filter options + final case class Terminate(replyTo: ActorRef[OsmoGridGuardian.Request]) + extends Request - def apply(cfg: OsmoGridConfig.Input): Behavior[InputDataEvent] = + sealed trait Response + + def apply(cfg: OsmoGridConfig.Input): Behavior[Request] = Behaviors.receive { (ctx, msg) => msg match { case _: Read => diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index 064ff7f9..32560ff6 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -14,18 +14,17 @@ import edu.ie3.datamodel.models.input.container.{ JointGridContainer } import edu.ie3.osmogrid.cfg.OsmoGridConfig -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.OsmoGridGuardianEvent +import edu.ie3.osmogrid.guardian.OsmoGridGuardian object ResultListener { - sealed trait ResultEvent - + sealed trait Request final case class GridResult( grid: JointGridContainer, - replyTo: ActorRef[OsmoGridGuardianEvent] - ) extends ResultEvent + replyTo: ActorRef[OsmoGridGuardian.Request] + ) extends Request - def apply(cfg: OsmoGridConfig.Output): Behavior[ResultEvent] = + def apply(cfg: OsmoGridConfig.Output): Behavior[Request] = Behaviors.receive { (ctx, msg) => msg match { case GridResult(grid, _) => diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala index 1889207e..d0072cb5 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala @@ -11,65 +11,64 @@ import akka.actor.typed.scaladsl.{Behaviors, Routers} import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.cfg.OsmoGridConfig.Generation.Lv -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ - OsmoGridGuardianEvent, - RepLvGrids -} +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RepLvGrids import edu.ie3.osmogrid.lv.LvGenerator object LvCoordinator { - sealed trait LvCoordinatorEvent + sealed trait Request final case class ReqLvGrids( cfg: OsmoGridConfig.Generation.Lv, - replyTo: ActorRef[OsmoGridGuardianEvent] - ) extends LvCoordinatorEvent + replyTo: ActorRef[Response] + ) extends Request + + sealed trait Response + final case class RepLvGrids(grids: Seq[SubGridContainer]) extends Response - def apply(): Behavior[LvCoordinatorEvent] = idle + def apply(): Behavior[Request] = idle - private def idle: Behavior[LvCoordinatorEvent] = Behaviors.receive { - (ctx, msg) => - msg match { - case ReqLvGrids( - Lv( - amountOfGridGenerators, - amountOfRegionCoordinators, - distinctHouseConnections - ), - replyTo - ) => - ctx.log.info("Starting generation of low voltage grids!") - /* TODO: + private def idle: Behavior[Request] = Behaviors.receive { (ctx, msg) => + msg match { + case ReqLvGrids( + Lv( + amountOfGridGenerators, + amountOfRegionCoordinators, + distinctHouseConnections + ), + replyTo + ) => + ctx.log.info("Starting generation of low voltage grids!") + /* TODO: 1) Ask for OSM data 2) Ask for asset data 3) Split up osm data at municipality boundaries 4) start generation */ - /* Spawn a pool of workers to build grids from sub-graphs */ - val lvGeneratorPool = - Routers.pool(poolSize = amountOfGridGenerators) { - // Restart workers on failure - Behaviors - .supervise(LvGenerator()) - .onFailure(SupervisorStrategy.restart) - } - val lvGeneratorProxy = ctx.spawn(lvGeneratorPool, "LvGeneratorPool") + /* Spawn a pool of workers to build grids from sub-graphs */ + val lvGeneratorPool = + Routers.pool(poolSize = amountOfGridGenerators) { + // Restart workers on failure + Behaviors + .supervise(LvGenerator()) + .onFailure(SupervisorStrategy.restart) + } + val lvGeneratorProxy = ctx.spawn(lvGeneratorPool, "LvGeneratorPool") - /* Spawn a pool of workers to build grids for one municipality */ - val lvRegionCoordinatorPool = - Routers.pool(poolSize = amountOfRegionCoordinators) { - // Restart workers on failure - Behaviors - .supervise(LvRegionCoordinator(lvGeneratorProxy)) - .onFailure(SupervisorStrategy.restart) - } - val lvRegionCoordinatorProxy = - ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") + /* Spawn a pool of workers to build grids for one municipality */ + val lvRegionCoordinatorPool = + Routers.pool(poolSize = amountOfRegionCoordinators) { + // Restart workers on failure + Behaviors + .supervise(LvRegionCoordinator(lvGeneratorProxy)) + .onFailure(SupervisorStrategy.restart) + } + val lvRegionCoordinatorProxy = + ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") - replyTo ! RepLvGrids(Vector.empty[SubGridContainer]) - Behaviors.stopped - case unsupported => - ctx.log.error(s"Received unsupported message: $unsupported") - Behaviors.stopped - } + replyTo ! RepLvGrids(Vector.empty[SubGridContainer]) + Behaviors.stopped + case unsupported => + ctx.log.error(s"Received unsupported message: $unsupported") + Behaviors.stopped + } } } diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvGenerator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvGenerator.scala index 56f3bb16..3d5d9abd 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvGenerator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvGenerator.scala @@ -9,11 +9,11 @@ package edu.ie3.osmogrid.lv import akka.actor.typed.scaladsl.Behaviors object LvGenerator { - sealed trait LvGeneratorEvent + sealed trait Request - def apply(): Behaviors.Receive[LvGeneratorEvent] = idle + def apply(): Behaviors.Receive[Request] = idle - private def idle: Behaviors.Receive[LvGeneratorEvent] = Behaviors.receive { + private def idle: Behaviors.Receive[Request] = Behaviors.receive { case (ctx, unsupported) => ctx.log.warn(s"Received unsupported message '$unsupported'.") Behaviors.stopped diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala index 55da4bed..25b53b8e 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala @@ -9,20 +9,18 @@ package edu.ie3.osmogrid.lv import akka.actor.typed.scaladsl.Behaviors import akka.actor.typed.ActorRef -import edu.ie3.osmogrid.lv.LvGenerator.LvGeneratorEvent object LvRegionCoordinator { - sealed trait LvRegionCoordinatorEvent + sealed trait Request def apply( - lvGeneratorPool: ActorRef[LvGeneratorEvent] - ): Behaviors.Receive[LvRegionCoordinatorEvent] = idle(lvGeneratorPool) + lvGeneratorPool: ActorRef[LvGenerator.Request] + ): Behaviors.Receive[Request] = idle(lvGeneratorPool) private def idle( - lvGeneratorPool: ActorRef[LvGeneratorEvent] - ): Behaviors.Receive[LvRegionCoordinatorEvent] = Behaviors.receive { - case (ctx, unsupported) => - ctx.log.warn(s"Received unsupported message '$unsupported'.") - Behaviors.stopped + lvGeneratorPool: ActorRef[LvGenerator.Request] + ): Behaviors.Receive[Request] = Behaviors.receive { case (ctx, unsupported) => + ctx.log.warn(s"Received unsupported message '$unsupported'.") + Behaviors.stopped } } diff --git a/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala b/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala index 71b23f97..17dca710 100644 --- a/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala +++ b/src/main/scala/edu/ie3/osmogrid/main/RunOsmoGridStandalone.scala @@ -8,7 +8,7 @@ package edu.ie3.osmogrid.main import akka.actor.typed.ActorSystem import edu.ie3.osmogrid.guardian.OsmoGridGuardian -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{OsmoGridGuardianEvent, Run} +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.Run import edu.ie3.osmogrid.cfg.{ArgsParser, ConfigFailFast, OsmoGridConfig} object RunOsmoGridStandalone { @@ -17,7 +17,7 @@ object RunOsmoGridStandalone { val cfg: OsmoGridConfig = ArgsParser.prepare(args) ConfigFailFast.check(cfg) - val actorSystem: ActorSystem[OsmoGridGuardianEvent] = + val actorSystem: ActorSystem[OsmoGridGuardian.Request] = ActorSystem(OsmoGridGuardian(), "OSMoGridGuardian") actorSystem ! Run(cfg) } From 9e2799fc58210da0497f57702bbd6f67e3dd2eff Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 17:01:32 +0100 Subject: [PATCH 02/38] Add support for run id --- .../osmogrid/guardian/OsmoGridGuardian.scala | 29 +++++++++++++++++-- .../osmogrid/io/output/ResultListener.scala | 2 ++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 7052127c..490b57f5 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -22,6 +22,7 @@ import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, Request} import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids +import java.util.UUID import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} @@ -29,13 +30,22 @@ object OsmoGridGuardian { /* Messages, that are understood and sent */ sealed trait Request - final case class Run(cfg: OsmoGridConfig) extends Request + final case class Run( + cfg: OsmoGridConfig, + additionalListener: Vector[ActorRef[ResultListener.ResultEvent]] = + Vector.empty, + runId: UUID = UUID.randomUUID() + ) extends Request + @Deprecated("Use child classes of GuardianWatch") object InputDataProviderDied extends Request + @Deprecated("Use child classes of GuardianWatch") object ResultEventListenerDied extends Request + @Deprecated("Use child classes of GuardianWatch") object LvCoordinatorDied extends Request sealed trait Response - final case class RepLvGrids(grids: Seq[SubGridContainer]) extends Response + final case class RepLvGrids(runId: UUID, grids: Seq[SubGridContainer]) + extends Response /* Container class for message adapters as well as wrapping classes themselves */ private final case class MessageAdapters( @@ -47,6 +57,19 @@ object OsmoGridGuardian { ) extends Request } + // dead watch events + sealed trait GuardianWatch extends Request { + val runId: UUID + } + + private final case class InputDataProviderDied(runId: UUID) + extends GuardianWatch + + private final case class ResultEventListenerDied(runId: UUID) + extends GuardianWatch + + private final case class LvCoordinatorDied(runId: UUID) extends GuardianWatch + def apply(): Behavior[Request] = Behaviors.setup { context => /* Define message adapters */ val messageAdapters = @@ -61,7 +84,7 @@ object OsmoGridGuardian { private def idle(msgAdapters: MessageAdapters): Behavior[Request] = Behaviors.receive { - case (ctx, Run(cfg)) => + case (ctx, Run(cfg, additionalListener, runId)) => ctx.log.info("Initializing grid generation!") ctx.log.info("Starting input data provider") diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index 32560ff6..a411a842 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -17,6 +17,8 @@ import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.guardian.OsmoGridGuardian object ResultListener { + /* internal API */ + sealed trait ResultEvent sealed trait Request final case class GridResult( From 355ce4edcd02f68cea8d8270bbbf032d0b10a049 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 17:24:03 +0100 Subject: [PATCH 03/38] Add actor data and run meta data Co-authored-by: johanneshiry --- .../osmogrid/guardian/OsmoGridGuardian.scala | 92 ++++++++++++++++--- .../osmogrid/io/output/ResultListener.scala | 8 +- 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 490b57f5..5931f898 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -47,17 +47,7 @@ object OsmoGridGuardian { final case class RepLvGrids(runId: UUID, grids: Seq[SubGridContainer]) extends Response - /* Container class for message adapters as well as wrapping classes themselves */ - private final case class MessageAdapters( - lvCoordinator: ActorRef[LvCoordinator.Response] - ) - private object MessageAdapters { - final case class WrappedLvCoordinatorResponse( - response: LvCoordinator.Response - ) extends Request - } - - // dead watch events + /* dead watch events */ sealed trait GuardianWatch extends Request { val runId: UUID } @@ -70,12 +60,90 @@ object OsmoGridGuardian { private final case class LvCoordinatorDied(runId: UUID) extends GuardianWatch + /** Relevant, state-independent data, the the actor needs to know + * + * @param msgAdapters + * Collection of all message adapters + * @param runs + * Currently active conversion runs + * @param persistenceResponseMapper + * Adapter for messages from [[ResultListener]] + */ + private final case class GuardianData( + msgAdapters: MessageAdapters, + runs: Map[UUID, RunData], + @Deprecated("Use the entry in msgAdapters") + persistenceResponseMapper: ActorRef[ResultListener.Response] + ) { + + def append(run: RunData): GuardianData = + this.copy(runs = runs + (run.runId -> run)) + + def remove(runId: UUID): GuardianData = + this.copy(runs = runs - runId) + } + + /** Container object with all available adapters for outside protocol messages + * + * @param lvCoordinator + * Adapter for messages from [[LvCoordinator]] + * @param resultListener + * Adapter for messages from [[ResultEventListener]] + */ + private final case class MessageAdapters( + lvCoordinator: ActorRef[LvCoordinator.Response], + resultListener: ActorRef[ResultListener.Response] + ) + private object MessageAdapters { + final case class WrappedLvCoordinatorResponse( + response: LvCoordinator.Response + ) extends Request + + final case class WrappedListenerResponse( + response: ResultListener.Response + ) extends Request + } + + /** Meta data regarding a certain given generation run + * + * @param runId + * Identifier of the run + * @param cfg + * Configuration for that given run + * @param resultEventListener + * Listeners to inform about results + * @param inputDataProvider + * Reference to the input data provider + */ + private final case class RunData( + runId: UUID, + cfg: OsmoGridConfig, + resultEventListener: Vector[ActorRef[ResultListener.ResultEvent]], + inputDataProvider: ActorRef[InputDataProvider.Request] + ) + private case object RunData { + def apply( + run: Run, + resultEventListener: Vector[ActorRef[ResultListener.ResultEvent]], + inputDataProvider: ActorRef[InputDataProvider.Request] + ): RunData = + RunData( + run.runId, + run.cfg, + run.additionalListener ++ resultEventListener, + inputDataProvider + ) + } + def apply(): Behavior[Request] = Behaviors.setup { context => - /* Define message adapters */ + /* Define and register message adapters */ val messageAdapters = MessageAdapters( context.messageAdapter(msg => MessageAdapters.WrappedLvCoordinatorResponse(msg) + ), + context.messageAdapter(msg => + MessageAdapters.WrappedListenerResponse(msg) ) ) diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index a411a842..884b56d8 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -17,15 +17,17 @@ import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.guardian.OsmoGridGuardian object ResultListener { - /* internal API */ - sealed trait ResultEvent - sealed trait Request final case class GridResult( grid: JointGridContainer, replyTo: ActorRef[OsmoGridGuardian.Request] ) extends Request + sealed trait Response + + /* internal API */ + sealed trait ResultEvent + def apply(cfg: OsmoGridConfig.Output): Behavior[Request] = Behaviors.receive { (ctx, msg) => msg match { From 1f524bf57cc37179e055b10660a0a24f17632225 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 17:47:14 +0100 Subject: [PATCH 04/38] Initiate a run and register its meta information Co-authored-by: johanneshiry --- .../resources/config/config-template.conf | 3 +- .../edu/ie3/osmogrid/cfg/ConfigFailFast.scala | 31 +++- .../edu/ie3/osmogrid/cfg/OsmoGridConfig.scala | 32 ++-- .../osmogrid/guardian/OsmoGridGuardian.scala | 154 ++++++++---------- .../osmogrid/io/output/ResultListener.scala | 16 +- .../edu/ie3/osmogrid/lv/LvCoordinator.scala | 4 + 6 files changed, 118 insertions(+), 122 deletions(-) diff --git a/src/main/resources/config/config-template.conf b/src/main/resources/config/config-template.conf index 21e5f74f..55ffa25f 100644 --- a/src/main/resources/config/config-template.conf +++ b/src/main/resources/config/config-template.conf @@ -13,8 +13,9 @@ input.asset: { } output: { #@optional - file: { + csv: { directory: String + separator: String | ";" hierarchic: Boolean | false } } diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala index 5c422184..5137d677 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala @@ -6,18 +6,25 @@ package edu.ie3.osmogrid.cfg +import akka.actor.typed.ActorRef +import com.typesafe.scalalogging.LazyLogging import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Input, Output} import edu.ie3.osmogrid.cfg.OsmoGridConfig.Input.{Asset, Osm} import edu.ie3.osmogrid.cfg.OsmoGridConfig.Generation import edu.ie3.osmogrid.cfg.OsmoGridConfig.Generation.Lv import edu.ie3.osmogrid.cfg.OsmoGridConfig.Input.Asset.File import edu.ie3.osmogrid.exception.IllegalConfigException +import edu.ie3.osmogrid.io.output.ResultListener -object ConfigFailFast { - def check(cfg: OsmoGridConfig): Unit = cfg match { +object ConfigFailFast extends LazyLogging { + def check( + cfg: OsmoGridConfig, + additionalListener: Vector[ActorRef[ResultListener.ResultEvent]] = + Vector.empty + ): Unit = cfg match { case OsmoGridConfig(generation, input, output) => checkInputConfig(input) - checkOutputConfig(output) + checkOutputConfig(output, additionalListener) checkGenerationConfig(generation) } @@ -89,20 +96,30 @@ object ConfigFailFast { case _ => /* I don't care. Everything is fine. */ } - private def checkOutputConfig(output: OsmoGridConfig.Output): Unit = + private def checkOutputConfig( + output: OsmoGridConfig.Output, + additionalListener: Vector[ActorRef[ResultListener.ResultEvent]] + ): Unit = output match { case Output(Some(file)) => checkOutputFile(file) + case Output(None) if additionalListener.nonEmpty => + logger.info( + "No output data type defined, but other listener provided. Will use them accordingly!" + ) case Output(None) => throw IllegalConfigException( "You have to provide at least one output data type!" ) } - private def checkOutputFile(file: OsmoGridConfig.Output.File): Unit = + private def checkOutputFile(file: OsmoGridConfig.Output.Csv): Unit = file match { - case OsmoGridConfig.Output.File(directory, _) if directory.isEmpty => - throw IllegalConfigException("Output directory may be set!") + case OsmoGridConfig.Output.Csv(directory, _, separator) + if directory.isEmpty || separator.isEmpty => + throw IllegalConfigException( + "Output directory and separator must be set when using .csv file sink!" + ) case _ => /* I don't care. Everything is fine. */ } } diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/OsmoGridConfig.scala b/src/main/scala/edu/ie3/osmogrid/cfg/OsmoGridConfig.scala index 650e6c53..5dcd3a1f 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/OsmoGridConfig.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/OsmoGridConfig.scala @@ -1,5 +1,5 @@ /* - * © 2021. TU Dortmund University, + * © 2022. TU Dortmund University, * Institute of Energy Systems, Energy Efficiency and Energy Economics, * Research group Distribution grid planning and operation */ @@ -35,7 +35,7 @@ object OsmoGridConfig { amountOfRegionCoordinators = if (c.hasPathOrNull("amountOfRegionCoordinators")) c.getInt("amountOfRegionCoordinators") - else 50, + else 5, distinctHouseConnections = c.hasPathOrNull( "distinctHouseConnections" ) && c.getBoolean("distinctHouseConnections") @@ -198,23 +198,26 @@ object OsmoGridConfig { } final case class Output( - file: scala.Option[OsmoGridConfig.Output.File] + csv: scala.Option[OsmoGridConfig.Output.Csv] ) object Output { - final case class File( + final case class Csv( directory: java.lang.String, - hierarchic: scala.Boolean + hierarchic: scala.Boolean, + separator: java.lang.String ) - object File { + object Csv { def apply( c: com.typesafe.config.Config, parentPath: java.lang.String, $tsCfgValidator: $TsCfgValidator - ): OsmoGridConfig.Output.File = { - OsmoGridConfig.Output.File( + ): OsmoGridConfig.Output.Csv = { + OsmoGridConfig.Output.Csv( directory = $_reqStr(parentPath, c, "directory", $tsCfgValidator), hierarchic = - c.hasPathOrNull("hierarchic") && c.getBoolean("hierarchic") + c.hasPathOrNull("hierarchic") && c.getBoolean("hierarchic"), + separator = + if (c.hasPathOrNull("separator")) c.getString("separator") else ";" ) } private def $_reqStr( @@ -241,14 +244,11 @@ object OsmoGridConfig { $tsCfgValidator: $TsCfgValidator ): OsmoGridConfig.Output = { OsmoGridConfig.Output( - file = - if (c.hasPathOrNull("file")) + csv = + if (c.hasPathOrNull("csv")) scala.Some( - OsmoGridConfig.Output.File( - c.getConfig("file"), - parentPath + "file.", - $tsCfgValidator - ) + OsmoGridConfig.Output + .Csv(c.getConfig("csv"), parentPath + "csv.", $tsCfgValidator) ) else None ) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 5931f898..b8544df3 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -7,15 +7,16 @@ package edu.ie3.osmogrid.guardian import akka.actor.typed.Behavior -import akka.actor.typed.scaladsl.Behaviors +import akka.actor.typed.scaladsl.{ActorContext, Behaviors} import akka.actor.typed.ActorRef import edu.ie3.datamodel.models.input.container.{ JointGridContainer, SubGridContainer } import edu.ie3.datamodel.utils.ContainerUtils -import edu.ie3.osmogrid.cfg.OsmoGridConfig -import edu.ie3.osmogrid.cfg.OsmoGridConfig.Generation +import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} +import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.stopChildrenByRun import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, Request} @@ -36,12 +37,6 @@ object OsmoGridGuardian { Vector.empty, runId: UUID = UUID.randomUUID() ) extends Request - @Deprecated("Use child classes of GuardianWatch") - object InputDataProviderDied extends Request - @Deprecated("Use child classes of GuardianWatch") - object ResultEventListenerDied extends Request - @Deprecated("Use child classes of GuardianWatch") - object LvCoordinatorDied extends Request sealed trait Response final case class RepLvGrids(runId: UUID, grids: Seq[SubGridContainer]) @@ -66,14 +61,10 @@ object OsmoGridGuardian { * Collection of all message adapters * @param runs * Currently active conversion runs - * @param persistenceResponseMapper - * Adapter for messages from [[ResultListener]] */ private final case class GuardianData( msgAdapters: MessageAdapters, - runs: Map[UUID, RunData], - @Deprecated("Use the entry in msgAdapters") - persistenceResponseMapper: ActorRef[ResultListener.Response] + runs: Map[UUID, RunData] ) { def append(run: RunData): GuardianData = @@ -147,94 +138,77 @@ object OsmoGridGuardian { ) ) - idle(messageAdapters) + idle(GuardianData(messageAdapters, Map.empty)) } - private def idle(msgAdapters: MessageAdapters): Behavior[Request] = + private def idle(guardianData: GuardianData): Behavior[Request] = Behaviors.receive { - case (ctx, Run(cfg, additionalListener, runId)) => - ctx.log.info("Initializing grid generation!") - - ctx.log.info("Starting input data provider") - val inputProvider = - ctx.spawn(InputDataProvider(cfg.input), "InputDataProvider") - ctx.watchWith(inputProvider, InputDataProviderDied) - ctx.log.debug("Starting output data listener") - val resultEventListener = - ctx.spawn(ResultListener(cfg.output), "ResultListener") - ctx.watchWith(resultEventListener, ResultEventListenerDied) - - /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ - cfg.generation match { - case Generation(Some(lvConfig)) => - ctx.log.debug("Starting low voltage grid coordinator.") - val lvCoordinator = ctx.spawn(LvCoordinator(), "LvCoordinator") - ctx.watchWith(lvCoordinator, LvCoordinatorDied) - lvCoordinator ! ReqLvGrids(lvConfig, msgAdapters.lvCoordinator) - awaitLvGrids(inputProvider, resultEventListener) - case unsupported => - ctx.log.error( - "Received unsupported grid generation config. Bye, bye." - ) - Behaviors.stopped - } + case (ctx, run: Run) => + val runData = initRun(run, ctx, guardianData.msgAdapters.lvCoordinator) + idle(guardianData.append(runData)) case (ctx, unsupported) => ctx.log.error(s"Received unsupported message '$unsupported'.") Behaviors.stopped } - private def awaitLvGrids( - inputDataProvider: ActorRef[InputDataProvider.Request], - resultListener: ActorRef[ResultListener.Request] - ): Behaviors.Receive[Request] = - Behaviors.receive { - case ( - ctx, - MessageAdapters.WrappedLvCoordinatorResponse( - LvCoordinator.RepLvGrids(lvGrids) - ) - ) => - ctx.log.info(s"Received ${lvGrids.length} lv grids. Join them.") - Try(ContainerUtils.combineToJointGrid(lvGrids.asJava)) match { - case Success(jointGrid) => - resultListener ! GridResult(jointGrid, ctx.self) - awaitShutDown(inputDataProvider) - case Failure(exception) => - ctx.log.error( - "Combination of received sub-grids failed. Shutting down." - ) - Behaviors.stopped - } - case (ctx, unsupported) => - ctx.log.error( - s"Received unsupported message while waiting for lv grids. Unsupported: $unsupported" + private def initRun( + run: Run, + ctx: ActorContext[Request], + lvCoordinatorAdapter: ActorRef[LvCoordinator.Response] + ): RunData = { + val log = ctx.log + ConfigFailFast.check(run.cfg, run.additionalListener) + log.info(s"Initializing grid generation for run with id '${run.runId}'!") + + log.info("Starting input data provider ...") + val inputProvider = + ctx.spawn(InputDataProvider(run.cfg.input), "InputDataProvider") + ctx.watchWith(inputProvider, InputDataProviderDied(run.runId)) + val resultEventListener = run.cfg.output match { + case Output(Some(_)) => + log.info("Starting output data listener ...") + Vector( + ctx.spawn( + ResultListener(run.runId, run.cfg.output), + "PersistenceResultListener" + ) ) - Behaviors.stopped + case Output(None) => + Vector.empty } - - private def awaitShutDown( - inputDataProvider: ActorRef[InputDataProvider.Request], - resultListenerTerminated: Boolean = false, - inputDataProviderTerminated: Boolean = false - ): Behaviors.Receive[Request] = Behaviors.receive { (ctx, msg) => - msg match { - case ResultEventListenerDied => - ctx.log.info("Result listener finished handling the result.") - ctx.log.debug("Shut down input data provider.") - awaitShutDown(inputDataProvider, resultListenerTerminated = true) - case InputDataProviderDied if resultListenerTerminated => - /* That's the fine case */ - ctx.log.info("Input data provider shut down.") - Behaviors.stopped - case InputDataProviderDied => - /* That's the malicious case */ + resultEventListener.foreach( + ctx.watchWith(_, ResultEventListenerDied(run.runId)) + ) + + /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ + run.cfg.generation match { + case Generation(Some(lvConfig)) => + ctx.log.info("Starting low voltage grid coordinator ...") + val lvCoordinator = ctx.spawn(LvCoordinator(), "LvCoordinator") + ctx.watchWith(lvCoordinator, LvCoordinatorDied(run.runId)) + ctx.log.info("Starting voltage level grid generation ...") + lvCoordinator ! ReqLvGrids(run.runId, lvConfig, lvCoordinatorAdapter) + case unsupported => ctx.log.error( - "Input data provider unexpectedly died during shutdown was initiated." + s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${run.runId}'!" + ) + stopChildrenByRun( + RunData(run, resultEventListener, inputProvider), + ctx ) - Behaviors.stopped - case unsupported => - ctx.log.error(s"Received an unsupported message $unsupported.") - Behaviors.stopped } + + RunData(run, resultEventListener, inputProvider) } + + private def stopChildrenByRun( + runData: RunData, + ctx: ActorContext[Request] + ): Unit = { + ctx.unwatch(runData.inputDataProvider) + ctx.stop(runData.inputDataProvider) + runData.resultEventListener.foreach(ctx.unwatch) + runData.resultEventListener.foreach(ctx.stop) + } + } diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index 884b56d8..973f1bff 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -16,26 +16,26 @@ import edu.ie3.datamodel.models.input.container.{ import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.guardian.OsmoGridGuardian +import java.util.UUID + object ResultListener { sealed trait Request final case class GridResult( grid: JointGridContainer, replyTo: ActorRef[OsmoGridGuardian.Request] ) extends Request + with ResultEvent sealed trait Response /* internal API */ sealed trait ResultEvent - def apply(cfg: OsmoGridConfig.Output): Behavior[Request] = - Behaviors.receive { (ctx, msg) => - msg match { - case GridResult(grid, _) => - ctx.log.info(s"Received grid result for grid '${grid.getGridName}'") - // TODO: Actual persistence and stuff, closing sinks, ... - Behaviors.stopped - } + def apply(runId: UUID, cfg: OsmoGridConfig.Output): Behavior[ResultEvent] = + Behaviors.receive { case (ctx, GridResult(grid, _)) => + ctx.log.info(s"Received grid result for grid '${grid.getGridName}'") + // TODO: Actual persistence and stuff, closing sinks, ... + Behaviors.stopped } } diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala index d0072cb5..1699978a 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala @@ -14,9 +14,12 @@ import edu.ie3.osmogrid.cfg.OsmoGridConfig.Generation.Lv import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RepLvGrids import edu.ie3.osmogrid.lv.LvGenerator +import java.util.UUID + object LvCoordinator { sealed trait Request final case class ReqLvGrids( + runId: UUID, cfg: OsmoGridConfig.Generation.Lv, replyTo: ActorRef[Response] ) extends Request @@ -29,6 +32,7 @@ object LvCoordinator { private def idle: Behavior[Request] = Behaviors.receive { (ctx, msg) => msg match { case ReqLvGrids( + runId, Lv( amountOfGridGenerators, amountOfRegionCoordinators, From 686b2b477700057dfca1b88350561ff0ed1a5a59 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 18:07:12 +0100 Subject: [PATCH 05/38] Refactoring run init process --- .../osmogrid/guardian/InitRunSupport.scala | 179 ++++++++++++++++++ .../osmogrid/guardian/OsmoGridGuardian.scala | 75 +------- 2 files changed, 186 insertions(+), 68 deletions(-) create mode 100644 src/main/scala/edu/ie3/osmogrid/guardian/InitRunSupport.scala diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/InitRunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/InitRunSupport.scala new file mode 100644 index 00000000..f909d387 --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/guardian/InitRunSupport.scala @@ -0,0 +1,179 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian + +import akka.actor.typed.ActorRef +import akka.actor.typed.scaladsl.ActorContext +import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} +import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ + InputDataProviderDied, + LvCoordinatorDied, + Request, + ResultEventListenerDied, + Run, + RunData +} +import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener +import edu.ie3.osmogrid.lv.LvCoordinator +import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids + +import java.util.UUID + +trait InitRunSupport { + + /** Initiate a generation run and return the updated run meta data + * + * @param run + * Current run meta data + * @param ctx + * Current actor context + * @param lvCoordinatorAdapter + * Message adapter to understand [[LvCoordinator]] + * @return + * Updated run meta data + */ + protected def initRun( + run: Run, + ctx: ActorContext[Request], + lvCoordinatorAdapter: ActorRef[LvCoordinator.Response] + ): RunData = { + val log = ctx.log + ConfigFailFast.check(run.cfg, run.additionalListener) + log.info(s"Initializing grid generation for run with id '${run.runId}'!") + + val inputProvider = + spawnInputDataProvider(run.runId, run.cfg.input, ctx) + val resultEventListener = + spawnResultListener(run.runId, run.cfg.output, ctx) + /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ + run.cfg.generation match { + case Generation(Some(lvConfig)) => + ctx.log.info("Starting low voltage grid coordinator ...") + startLvGridGeneration( + run.runId, + lvConfig, + lvCoordinatorAdapter, + ctx + ) + case unsupported => + ctx.log.error( + s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${run.runId}'!" + ) + stopChildrenByRun( + RunData(run, resultEventListener, inputProvider), + ctx + ) + } + + RunData(run, resultEventListener, inputProvider) + } + + /** Spawn an input data provider for this run + * + * @param runId + * Identifier for the targeted run + * @param inputConfig + * Configuration for the input behavior + * @param ctx + * Current actor context + * @return + * Reference to an [[InputDataProvider]] + */ + private def spawnInputDataProvider( + runId: UUID, + inputConfig: OsmoGridConfig.Input, + ctx: ActorContext[Request] + ): ActorRef[InputDataProvider.Request] = { + ctx.log.info("Starting input data provider ...") + val inputProvider = + ctx.spawn( + InputDataProvider(inputConfig), + s"InputDataProvider_${runId.toString}" + ) + ctx.watchWith(inputProvider, InputDataProviderDied(runId)) + inputProvider + } + + /** Spawn a result listener for the specified run + * + * @param runId + * Identifier for the targeted run + * @param outputConfig + * Configuration of the output behavior + * @param ctx + * Current actor context + * @return + * References to [[ResultListener]] + */ + private def spawnResultListener( + runId: UUID, + outputConfig: OsmoGridConfig.Output, + ctx: ActorContext[Request] + ): Vector[ActorRef[ResultListener.ResultEvent]] = { + val resultListener = outputConfig match { + case Output(Some(_)) => + ctx.log.info("Starting output data listener ...") + Vector( + ctx.spawn( + ResultListener(runId, outputConfig), + s"PersistenceResultListener_${runId.toString}" + ) + ) + case Output(None) => + Vector.empty + } + resultListener.foreach( + ctx.watchWith(_, ResultEventListenerDied(runId)) + ) + resultListener + } + + /** Spawn a [[LvCoordinator]] for the targeted run and ask it to start + * conversion + * + * @param runId + * Identifier for the targeted run + * @param lvConfig + * Configuration for low voltage grid generation + * @param lvCoordinatorAdapter + * Message adapter to understand responses from [[LvCoordinator]] + * @param ctx + * Current actor context + */ + private def startLvGridGeneration( + runId: UUID, + lvConfig: OsmoGridConfig.Generation.Lv, + lvCoordinatorAdapter: ActorRef[LvCoordinator.Response], + ctx: ActorContext[Request] + ): Unit = { + val lvCoordinator = + ctx.spawn(LvCoordinator(), s"LvCoordinator_${runId.toString}") + ctx.watchWith(lvCoordinator, LvCoordinatorDied(runId)) + + ctx.log.info("Starting voltage level grid generation ...") + lvCoordinator ! ReqLvGrids(runId, lvConfig, lvCoordinatorAdapter) + } + + /** Stop all children for the given run + * + * @param runData + * Current run meta data + * @param ctx + * Current actor context + */ + private def stopChildrenByRun( + runData: RunData, + ctx: ActorContext[Request] + ): Unit = { + ctx.unwatch(runData.inputDataProvider) + ctx.stop(runData.inputDataProvider) + runData.resultEventListener.foreach(ctx.unwatch) + runData.resultEventListener.foreach(ctx.stop) + } +} diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index b8544df3..b1071a6e 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -16,7 +16,6 @@ import edu.ie3.datamodel.models.input.container.{ import edu.ie3.datamodel.utils.ContainerUtils import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.stopChildrenByRun import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, Request} @@ -27,7 +26,7 @@ import java.util.UUID import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} -object OsmoGridGuardian { +object OsmoGridGuardian extends InitRunSupport { /* Messages, that are understood and sent */ sealed trait Request @@ -47,13 +46,14 @@ object OsmoGridGuardian { val runId: UUID } - private final case class InputDataProviderDied(runId: UUID) + private[guardian] final case class InputDataProviderDied(runId: UUID) extends GuardianWatch - private final case class ResultEventListenerDied(runId: UUID) + private[guardian] final case class ResultEventListenerDied(runId: UUID) extends GuardianWatch - private final case class LvCoordinatorDied(runId: UUID) extends GuardianWatch + private[guardian] final case class LvCoordinatorDied(runId: UUID) + extends GuardianWatch /** Relevant, state-independent data, the the actor needs to know * @@ -106,13 +106,13 @@ object OsmoGridGuardian { * @param inputDataProvider * Reference to the input data provider */ - private final case class RunData( + private[guardian] final case class RunData( runId: UUID, cfg: OsmoGridConfig, resultEventListener: Vector[ActorRef[ResultListener.ResultEvent]], inputDataProvider: ActorRef[InputDataProvider.Request] ) - private case object RunData { + private[guardian] case object RunData { def apply( run: Run, resultEventListener: Vector[ActorRef[ResultListener.ResultEvent]], @@ -150,65 +150,4 @@ object OsmoGridGuardian { ctx.log.error(s"Received unsupported message '$unsupported'.") Behaviors.stopped } - - private def initRun( - run: Run, - ctx: ActorContext[Request], - lvCoordinatorAdapter: ActorRef[LvCoordinator.Response] - ): RunData = { - val log = ctx.log - ConfigFailFast.check(run.cfg, run.additionalListener) - log.info(s"Initializing grid generation for run with id '${run.runId}'!") - - log.info("Starting input data provider ...") - val inputProvider = - ctx.spawn(InputDataProvider(run.cfg.input), "InputDataProvider") - ctx.watchWith(inputProvider, InputDataProviderDied(run.runId)) - val resultEventListener = run.cfg.output match { - case Output(Some(_)) => - log.info("Starting output data listener ...") - Vector( - ctx.spawn( - ResultListener(run.runId, run.cfg.output), - "PersistenceResultListener" - ) - ) - case Output(None) => - Vector.empty - } - resultEventListener.foreach( - ctx.watchWith(_, ResultEventListenerDied(run.runId)) - ) - - /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ - run.cfg.generation match { - case Generation(Some(lvConfig)) => - ctx.log.info("Starting low voltage grid coordinator ...") - val lvCoordinator = ctx.spawn(LvCoordinator(), "LvCoordinator") - ctx.watchWith(lvCoordinator, LvCoordinatorDied(run.runId)) - ctx.log.info("Starting voltage level grid generation ...") - lvCoordinator ! ReqLvGrids(run.runId, lvConfig, lvCoordinatorAdapter) - case unsupported => - ctx.log.error( - s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${run.runId}'!" - ) - stopChildrenByRun( - RunData(run, resultEventListener, inputProvider), - ctx - ) - } - - RunData(run, resultEventListener, inputProvider) - } - - private def stopChildrenByRun( - runData: RunData, - ctx: ActorContext[Request] - ): Unit = { - ctx.unwatch(runData.inputDataProvider) - ctx.stop(runData.inputDataProvider) - runData.resultEventListener.foreach(ctx.unwatch) - runData.resultEventListener.foreach(ctx.stop) - } - } From 20529ea2f02451892b655d2ff16df76b0a3fb929 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 18:13:38 +0100 Subject: [PATCH 06/38] Handle death watch events Co-authored-by: johanneshiry --- .../edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala | 9 ++++++++- .../guardian/{InitRunSupport.scala => RunSupport.scala} | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) rename src/main/scala/edu/ie3/osmogrid/guardian/{InitRunSupport.scala => RunSupport.scala} (98%) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index b1071a6e..0df0fa3d 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -26,7 +26,7 @@ import java.util.UUID import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} -object OsmoGridGuardian extends InitRunSupport { +object OsmoGridGuardian extends RunSupport { /* Messages, that are understood and sent */ sealed trait Request @@ -146,6 +146,13 @@ object OsmoGridGuardian extends InitRunSupport { case (ctx, run: Run) => val runData = initRun(run, ctx, guardianData.msgAdapters.lvCoordinator) idle(guardianData.append(runData)) + case (ctx, watch: GuardianWatch) => + ctx.log.error( + s"Received dead message '$watch' for run ${watch.runId}! " + + s"Stopping all corresponding children ..." + ) + guardianData.runs.get(watch.runId).foreach(stopChildrenByRun(_, ctx)) + idle(guardianData.remove(watch.runId)) case (ctx, unsupported) => ctx.log.error(s"Received unsupported message '$unsupported'.") Behaviors.stopped diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/InitRunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala similarity index 98% rename from src/main/scala/edu/ie3/osmogrid/guardian/InitRunSupport.scala rename to src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala index f909d387..d9e96b47 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/InitRunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala @@ -25,7 +25,7 @@ import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids import java.util.UUID -trait InitRunSupport { +trait RunSupport { /** Initiate a generation run and return the updated run meta data * @@ -167,7 +167,7 @@ trait InitRunSupport { * @param ctx * Current actor context */ - private def stopChildrenByRun( + protected def stopChildrenByRun( runData: RunData, ctx: ActorContext[Request] ): Unit = { From f039d1b5b9904560584e1ad1ffd00b4862bcdc06 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 18:44:29 +0100 Subject: [PATCH 07/38] Handle incoming results --- .../osmogrid/guardian/OsmoGridGuardian.scala | 14 ++-- .../osmogrid/guardian/SubGridHandling.scala | 77 +++++++++++++++++++ .../osmogrid/io/output/ResultListener.scala | 6 +- .../edu/ie3/osmogrid/lv/LvCoordinator.scala | 6 +- 4 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 0df0fa3d..2d11f8c1 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -26,7 +26,7 @@ import java.util.UUID import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} -object OsmoGridGuardian extends RunSupport { +object OsmoGridGuardian extends RunSupport with SubGridHandling { /* Messages, that are understood and sent */ sealed trait Request @@ -38,8 +38,6 @@ object OsmoGridGuardian extends RunSupport { ) extends Request sealed trait Response - final case class RepLvGrids(runId: UUID, grids: Seq[SubGridContainer]) - extends Response /* dead watch events */ sealed trait GuardianWatch extends Request { @@ -62,7 +60,7 @@ object OsmoGridGuardian extends RunSupport { * @param runs * Currently active conversion runs */ - private final case class GuardianData( + private[guardian] final case class GuardianData( msgAdapters: MessageAdapters, runs: Map[UUID, RunData] ) { @@ -81,7 +79,7 @@ object OsmoGridGuardian extends RunSupport { * @param resultListener * Adapter for messages from [[ResultEventListener]] */ - private final case class MessageAdapters( + private[guardian] final case class MessageAdapters( lvCoordinator: ActorRef[LvCoordinator.Response], resultListener: ActorRef[ResultListener.Response] ) @@ -146,6 +144,12 @@ object OsmoGridGuardian extends RunSupport { case (ctx, run: Run) => val runData = initRun(run, ctx, guardianData.msgAdapters.lvCoordinator) idle(guardianData.append(runData)) + case (ctx, MessageAdapters.WrappedLvCoordinatorResponse(response)) => + response match { + case LvCoordinator.RepLvGrids(runId, grids) => + handleLvResults(runId, grids, guardianData, ctx) + Behaviors.same + } case (ctx, watch: GuardianWatch) => ctx.log.error( s"Received dead message '$watch' for run ${watch.runId}! " + diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala new file mode 100644 index 00000000..2a099649 --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala @@ -0,0 +1,77 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian + +import akka.actor.typed.scaladsl.ActorContext +import edu.ie3.datamodel.models.input.container.SubGridContainer +import edu.ie3.datamodel.utils.ContainerUtils +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ + GuardianData, + Request, + RunData +} +import edu.ie3.osmogrid.io.output.ResultListener + +import java.util.UUID +import scala.jdk.CollectionConverters.* + +trait SubGridHandling { + + /** Handle incoming low voltage grid results + * + * @param runId + * Reference to the current run + * @param grids + * Received grids + * @param guardianData + * Relevant, state-independent data, the the actor needs to know + * @param ctx + * Current actor context + */ + protected def handleLvResults( + runId: UUID, + grids: Seq[SubGridContainer], + guardianData: GuardianData, + ctx: ActorContext[Request] + ): Unit = { + ctx.log.info("All lv grids successfully generated.") + val updatedSubGrids = assignSubnetNumbers(grids) + + guardianData.runs.get(runId) match { + case Some( + RunData(runId, cfg, resultEventListener, inputDataProvider) + ) => + // TODO: Check for mv config and issue run there, if applicable + ctx.log.debug( + "No further generation steps intended. Hand over results to result handler." + ) + /* Bundle grid result and inform interested listeners */ + val jointGrid = + ContainerUtils.combineToJointGrid(updatedSubGrids.asJava) + resultEventListener.foreach { listener => + listener ! ResultListener.GridResult( + jointGrid, + guardianData.msgAdapters.resultListener + ) + } + case None => + ctx.log.error( + s"Cannot find run information for '$runId', although it is supposed to be active. No further actions taken." + ) + } + } + + private def assignSubnetNumbers( + subnets: Seq[SubGridContainer] + ): Seq[SubGridContainer] = subnets.zipWithIndex.map { + case (subGrid, subnetNumber) => + assignSubnetNumber(subGrid, subnetNumber + 1) + } + + private def assignSubnetNumber(subGrid: SubGridContainer, subnetNumber: Int) = + subGrid +} diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index 973f1bff..57a3adff 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -21,8 +21,8 @@ import java.util.UUID object ResultListener { sealed trait Request final case class GridResult( - grid: JointGridContainer, - replyTo: ActorRef[OsmoGridGuardian.Request] + grid: GridContainer, + replyTo: ActorRef[Response] ) extends Request with ResultEvent @@ -33,7 +33,7 @@ object ResultListener { def apply(runId: UUID, cfg: OsmoGridConfig.Output): Behavior[ResultEvent] = Behaviors.receive { case (ctx, GridResult(grid, _)) => - ctx.log.info(s"Received grid result for grid '${grid.getGridName}'") + ctx.log.info(s"Received grid result for run id '${runId.toString}'") // TODO: Actual persistence and stuff, closing sinks, ... Behaviors.stopped } diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala index 1699978a..cb658fd6 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala @@ -11,7 +11,6 @@ import akka.actor.typed.scaladsl.{Behaviors, Routers} import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.cfg.OsmoGridConfig.Generation.Lv -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RepLvGrids import edu.ie3.osmogrid.lv.LvGenerator import java.util.UUID @@ -25,7 +24,8 @@ object LvCoordinator { ) extends Request sealed trait Response - final case class RepLvGrids(grids: Seq[SubGridContainer]) extends Response + final case class RepLvGrids(runId: UUID, grids: Seq[SubGridContainer]) + extends Response def apply(): Behavior[Request] = idle @@ -68,7 +68,7 @@ object LvCoordinator { val lvRegionCoordinatorProxy = ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") - replyTo ! RepLvGrids(Vector.empty[SubGridContainer]) + replyTo ! RepLvGrids(runId, Vector.empty[SubGridContainer]) Behaviors.stopped case unsupported => ctx.log.error(s"Received unsupported message: $unsupported") From e564e71c3ca84fb24bcb22e8535c3864f1c0969e Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 18:49:44 +0100 Subject: [PATCH 08/38] Address successful result handling --- .../osmogrid/guardian/OsmoGridGuardian.scala | 21 +++++++++++++++++-- .../osmogrid/io/output/ResultListener.scala | 6 ++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 2d11f8c1..9d6ea906 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -150,15 +150,32 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { handleLvResults(runId, grids, guardianData, ctx) Behaviors.same } + case (ctx, MessageAdapters.WrappedListenerResponse(response)) => + response match { + case ResultListener.ResultHandled(runId) => + ctx.log.info( + s"Results for run $runId handled successfully. Shutting down processes for this run." + ) + idle(stopRunProcesses(guardianData, runId, ctx)) + } case (ctx, watch: GuardianWatch) => ctx.log.error( s"Received dead message '$watch' for run ${watch.runId}! " + s"Stopping all corresponding children ..." ) - guardianData.runs.get(watch.runId).foreach(stopChildrenByRun(_, ctx)) - idle(guardianData.remove(watch.runId)) + + idle(stopRunProcesses(guardianData, watch.runId, ctx)) case (ctx, unsupported) => ctx.log.error(s"Received unsupported message '$unsupported'.") Behaviors.stopped } + + private def stopRunProcesses( + guardianData: GuardianData, + runId: UUID, + ctx: ActorContext[Request] + ): GuardianData = { + guardianData.runs.get(runId).foreach(stopChildrenByRun(_, ctx)) + guardianData.remove(runId) + } } diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index 57a3adff..86c50234 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -27,15 +27,17 @@ object ResultListener { with ResultEvent sealed trait Response + final case class ResultHandled(runId: UUID) extends Response /* internal API */ sealed trait ResultEvent def apply(runId: UUID, cfg: OsmoGridConfig.Output): Behavior[ResultEvent] = - Behaviors.receive { case (ctx, GridResult(grid, _)) => + Behaviors.receive { case (ctx, GridResult(grid, replyTo)) => ctx.log.info(s"Received grid result for run id '${runId.toString}'") // TODO: Actual persistence and stuff, closing sinks, ... - Behaviors.stopped + replyTo ! ResultHandled(runId) + Behaviors.same } } From a4bb7ded7a5620b9f10c9f70ec5db34d2b897344 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 13 Jan 2022 19:07:30 +0100 Subject: [PATCH 09/38] Fix broken test --- .../edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala b/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala index 2c26b19a..97f78401 100644 --- a/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala @@ -88,7 +88,7 @@ class ConfigFailFastSpec extends UnitSpec { """input.osm.pbf.file = "input_file_path" |input.asset.file.directory = "asset_input_dir" |input.asset.file.hierarchic = false - |output.file.directory = "output_file_path"""".stripMargin + |output.csv.directory = "output_file_path"""".stripMargin } match { case Success(cfg) => val exc = @@ -106,7 +106,7 @@ class ConfigFailFastSpec extends UnitSpec { """input.osm.pbf.file = "input_file_path" |input.asset.file.directory = "asset_input_dir" |input.asset.file.hierarchic = false - |output.file.directory = "output_file_path" + |output.csv.directory = "output_file_path" |generation.lv.amountOfGridGenerators = 0 |generation.lv.amountOfRegionCoordinators = 5 |generation.lv.distinctHouseConnections = false""".stripMargin @@ -125,7 +125,7 @@ class ConfigFailFastSpec extends UnitSpec { """input.osm.pbf.file = "input_file_path" |input.asset.file.directory = "asset_input_dir" |input.asset.file.hierarchic = false - |output.file.directory = "output_file_path" + |output.csv.directory = "output_file_path" |generation.lv.amountOfGridGenerators = -42 |generation.lv.amountOfRegionCoordinators = 5 |generation.lv.distinctHouseConnections = false""".stripMargin @@ -144,7 +144,7 @@ class ConfigFailFastSpec extends UnitSpec { """input.osm.pbf.file = "input_file_path" |input.asset.file.directory = "asset_input_dir" |input.asset.file.hierarchic = false - |output.file.directory = "output_file_path" + |output.csv.directory = "output_file_path" |generation.lv.amountOfGridGenerators = 10 |generation.lv.amountOfRegionCoordinators = 0 |generation.lv.distinctHouseConnections = false""".stripMargin @@ -163,7 +163,7 @@ class ConfigFailFastSpec extends UnitSpec { """input.osm.pbf.file = "input_file_path" |input.asset.file.directory = "asset_input_dir" |input.asset.file.hierarchic = false - |output.file.directory = "output_file_path" + |output.csv.directory = "output_file_path" |generation.lv.amountOfGridGenerators = 10 |generation.lv.amountOfRegionCoordinators = -42 |generation.lv.distinctHouseConnections = false""".stripMargin @@ -184,7 +184,7 @@ class ConfigFailFastSpec extends UnitSpec { """input.osm.pbf.file = "input_file_path" |input.asset.file.directory = "asset_input_dir" |input.asset.file.hierarchic = false - |output.file.directory = "output_file_path" + |output.csv.directory = "output_file_path" |generation.lv.distinctHouseConnections = true""".stripMargin } match { case Success(cfg) => From 8a5f1534ad75d99faf4e3018cd4109165db2cb58 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 17 Jan 2022 16:59:55 +0100 Subject: [PATCH 10/38] Addressing reviewer's comments --- .../edu/ie3/osmogrid/cfg/ConfigFailFast.scala | 15 +++- .../UnsupportedRequestException.scala | 12 ++++ .../osmogrid/guardian/OsmoGridGuardian.scala | 70 +++++++++++-------- .../ie3/osmogrid/guardian/RunSupport.scala | 49 +++++++++---- 4 files changed, 102 insertions(+), 44 deletions(-) create mode 100644 src/main/scala/edu/ie3/osmogrid/exception/UnsupportedRequestException.scala diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala index 5137d677..51f77745 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala @@ -77,7 +77,10 @@ object ConfigFailFast extends LazyLogging { file match { case File(directory, _) if directory.isEmpty => throw IllegalConfigException("Asset input directory may be set!") - case _ => /* I don't care. Everything is fine. */ + case unsupported => + throw IllegalConfigException( + s"Unable to run sanity checks on unsupported asset input file config '$unsupported'." + ) } private def checkOsmInputConfig(osm: OsmoGridConfig.Input.Osm): Unit = @@ -93,7 +96,10 @@ object ConfigFailFast extends LazyLogging { pbf match { case OsmoGridConfig.Input.Osm.Pbf(file) if file.isEmpty => throw IllegalConfigException("Pbf file may be set!") - case _ => /* I don't care. Everything is fine. */ + case unsupported => + throw IllegalConfigException( + s"Unable to run sanity checks on unsupported input file config '$unsupported'." + ) } private def checkOutputConfig( @@ -120,6 +126,9 @@ object ConfigFailFast extends LazyLogging { throw IllegalConfigException( "Output directory and separator must be set when using .csv file sink!" ) - case _ => /* I don't care. Everything is fine. */ + case unsupported => + throw IllegalConfigException( + s"Unable to run sanity checks on unsupported output file config '$unsupported'." + ) } } diff --git a/src/main/scala/edu/ie3/osmogrid/exception/UnsupportedRequestException.scala b/src/main/scala/edu/ie3/osmogrid/exception/UnsupportedRequestException.scala new file mode 100644 index 00000000..c142dcd4 --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/exception/UnsupportedRequestException.scala @@ -0,0 +1,12 @@ +/* + * © 2021. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.exception + +case class UnsupportedRequestException( + msg: String = "", + cause: Throwable = None.orNull +) extends Exception(msg, cause) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 9d6ea906..b41970b8 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -30,6 +30,16 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { /* Messages, that are understood and sent */ sealed trait Request + + /** Message to initiate a grid generation run + * + * @param cfg + * Configuration for the tool + * @param additionalListener + * Addresses of additional listeners to be informed about results + * @param runId + * Unique identifier for that generation run + */ final case class Run( cfg: OsmoGridConfig, additionalListener: Vector[ActorRef[ResultListener.ResultEvent]] = @@ -37,6 +47,27 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { runId: UUID = UUID.randomUUID() ) extends Request + /** Container object with all available adapters for outside protocol messages + * + * @param lvCoordinator + * Adapter for messages from [[LvCoordinator]] + * @param resultListener + * Adapter for messages from [[ResultEventListener]] + */ + private[guardian] final case class MessageAdapters( + lvCoordinator: ActorRef[LvCoordinator.Response], + resultListener: ActorRef[ResultListener.Response] + ) + private object MessageAdapters { + final case class WrappedLvCoordinatorResponse( + response: LvCoordinator.Response + ) extends Request + + final case class WrappedListenerResponse( + response: ResultListener.Response + ) extends Request + } + sealed trait Response /* dead watch events */ @@ -72,27 +103,6 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { this.copy(runs = runs - runId) } - /** Container object with all available adapters for outside protocol messages - * - * @param lvCoordinator - * Adapter for messages from [[LvCoordinator]] - * @param resultListener - * Adapter for messages from [[ResultEventListener]] - */ - private[guardian] final case class MessageAdapters( - lvCoordinator: ActorRef[LvCoordinator.Response], - resultListener: ActorRef[ResultListener.Response] - ) - private object MessageAdapters { - final case class WrappedLvCoordinatorResponse( - response: LvCoordinator.Response - ) extends Request - - final case class WrappedListenerResponse( - response: ResultListener.Response - ) extends Request - } - /** Meta data regarding a certain given generation run * * @param runId @@ -107,13 +117,13 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { private[guardian] final case class RunData( runId: UUID, cfg: OsmoGridConfig, - resultEventListener: Vector[ActorRef[ResultListener.ResultEvent]], + resultEventListener: Seq[ActorRef[ResultListener.ResultEvent]], inputDataProvider: ActorRef[InputDataProvider.Request] ) private[guardian] case object RunData { def apply( run: Run, - resultEventListener: Vector[ActorRef[ResultListener.ResultEvent]], + resultEventListener: Seq[ActorRef[ResultListener.ResultEvent]], inputDataProvider: ActorRef[InputDataProvider.Request] ): RunData = RunData( @@ -142,8 +152,15 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { private def idle(guardianData: GuardianData): Behavior[Request] = Behaviors.receive { case (ctx, run: Run) => - val runData = initRun(run, ctx, guardianData.msgAdapters.lvCoordinator) - idle(guardianData.append(runData)) + initRun(run, ctx, guardianData.msgAdapters.lvCoordinator) match { + case Success(runData) => idle(guardianData.append(runData)) + case Failure(exception) => + ctx.log.error( + "Issuing of run failed. Keep on going without that run.", + exception + ) + idle(guardianData) + } case (ctx, MessageAdapters.WrappedLvCoordinatorResponse(response)) => response match { case LvCoordinator.RepLvGrids(runId, grids) => @@ -165,9 +182,6 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { ) idle(stopRunProcesses(guardianData, watch.runId, ctx)) - case (ctx, unsupported) => - ctx.log.error(s"Received unsupported message '$unsupported'.") - Behaviors.stopped } private def stopRunProcesses( diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala index d9e96b47..f6bc28a4 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala @@ -24,6 +24,7 @@ import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids import java.util.UUID +import scala.util.{Try, Success, Failure} trait RunSupport { @@ -42,38 +43,59 @@ trait RunSupport { run: Run, ctx: ActorContext[Request], lvCoordinatorAdapter: ActorRef[LvCoordinator.Response] - ): RunData = { + ): Try[RunData] = { val log = ctx.log ConfigFailFast.check(run.cfg, run.additionalListener) log.info(s"Initializing grid generation for run with id '${run.runId}'!") - val inputProvider = - spawnInputDataProvider(run.runId, run.cfg.input, ctx) - val resultEventListener = - spawnResultListener(run.runId, run.cfg.output, ctx) /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ run.cfg.generation match { case Generation(Some(lvConfig)) => ctx.log.info("Starting low voltage grid coordinator ...") + val (inputProvider, resultEventListener) = + spawnIoActors(run.runId, run.cfg.input, run.cfg.output, ctx) startLvGridGeneration( run.runId, lvConfig, lvCoordinatorAdapter, ctx ) + Success(RunData(run, resultEventListener, inputProvider)) case unsupported => ctx.log.error( s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${run.runId}'!" ) - stopChildrenByRun( - RunData(run, resultEventListener, inputProvider), - ctx + Failure( + UnsupportedOperationException( + s"Unable to issue a generation run with the given parameters: '${run.cfg.generation}'" + ) ) } - - RunData(run, resultEventListener, inputProvider) } + /** Spawns both the input and the output actor for the given specific run + * + * @param runId + * Identifier for the targeted run + * @param inputConfig + * Configuration for the input behavior + * @param outputConfig + * Configuration of the output behavior + * @param ctx + * Current actor context + * @return + * Reference to an [[InputDataProvider]] as well as [[ResultListener]] + */ + private def spawnIoActors( + runId: UUID, + inputConfig: OsmoGridConfig.Input, + outputConfig: OsmoGridConfig.Output, + ctx: ActorContext[Request] + ) = ( + spawnInputDataProvider(runId, inputConfig, ctx), + spawnResultListener(runId, outputConfig, ctx) + ) + /** Spawn an input data provider for this run * * @param runId @@ -115,18 +137,19 @@ trait RunSupport { runId: UUID, outputConfig: OsmoGridConfig.Output, ctx: ActorContext[Request] - ): Vector[ActorRef[ResultListener.ResultEvent]] = { + ): Seq[ActorRef[ResultListener.ResultEvent]] = { val resultListener = outputConfig match { case Output(Some(_)) => ctx.log.info("Starting output data listener ...") - Vector( + Seq( ctx.spawn( ResultListener(runId, outputConfig), s"PersistenceResultListener_${runId.toString}" ) ) case Output(None) => - Vector.empty + ctx.log.warn(s"No result listener configured for run $runId.") + Seq.empty } resultListener.foreach( ctx.watchWith(_, ResultEventListenerDied(runId)) From 8186d5e9a649004aedd7c8313bc176409abfb9f4 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 08:58:44 +0100 Subject: [PATCH 11/38] Post stop phase for InputDataProvider and ResultListener --- .../osmogrid/io/input/InputDataProvider.scala | 16 ++++++++------ .../osmogrid/io/output/ResultListener.scala | 21 ++++++++++++------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala b/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala index 5cc866e0..edab8dfe 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala @@ -8,6 +8,7 @@ package edu.ie3.osmogrid.io.input import akka.actor.typed.Behavior import akka.actor.typed.ActorRef +import akka.actor.typed.PostStop import akka.actor.typed.scaladsl.Behaviors import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.guardian.OsmoGridGuardian @@ -23,16 +24,19 @@ object InputDataProvider { sealed trait Response def apply(cfg: OsmoGridConfig.Input): Behavior[Request] = - Behaviors.receive { (ctx, msg) => - msg match { - case _: Read => + Behaviors + .receive[Request] { + case (ctx, _: Read) => ctx.log.warn("Reading of data not yet implemented.") Behaviors.same - case Terminate(_) => + case (ctx, Terminate(_)) => ctx.log.info("Stopping input data provider") // TODO: Any closing of sources and stuff Behaviors.stopped } - } - + .receiveSignal { case (ctx, PostStop) => + ctx.log.info("Requested to stop.") + // TODO: Any closing of sources and stuff + Behaviors.same + } } diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index 86c50234..c9dfe83b 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -6,8 +6,7 @@ package edu.ie3.osmogrid.io.output -import akka.actor.typed.Behavior -import akka.actor.typed.ActorRef +import akka.actor.typed.{ActorRef, Behavior, PostStop} import akka.actor.typed.scaladsl.Behaviors import edu.ie3.datamodel.models.input.container.{ GridContainer, @@ -33,11 +32,17 @@ object ResultListener { sealed trait ResultEvent def apply(runId: UUID, cfg: OsmoGridConfig.Output): Behavior[ResultEvent] = - Behaviors.receive { case (ctx, GridResult(grid, replyTo)) => - ctx.log.info(s"Received grid result for run id '${runId.toString}'") - // TODO: Actual persistence and stuff, closing sinks, ... - replyTo ! ResultHandled(runId) - Behaviors.same - } + Behaviors + .receive[ResultEvent] { case (ctx, GridResult(grid, replyTo)) => + ctx.log.info(s"Received grid result for run id '${runId.toString}'") + // TODO: Actual persistence and stuff, closing sinks, ... + replyTo ! ResultHandled(runId) + Behaviors.same + } + .receiveSignal { case (ctx, PostStop) => + ctx.log.info("Requested to stop.") + // TODO: Any closing of sources and stuff + Behaviors.same + } } From c0b6cbe93812dc666470a6114ff2c85c0f8a81bd Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 09:38:58 +0100 Subject: [PATCH 12/38] Introduce coordinated shut down phase --- CHANGELOG.md | 7 +- .../osmogrid/guardian/OsmoGridGuardian.scala | 94 +++++---- .../ie3/osmogrid/guardian/RunSupport.scala | 17 -- .../ie3/osmogrid/guardian/StopSupport.scala | 193 ++++++++++++++++++ .../osmogrid/guardian/SubGridHandling.scala | 3 +- 5 files changed, 260 insertions(+), 54 deletions(-) create mode 100644 src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index e843fc5a..ff3cc068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased/Snapshot] +## [Unreleased] ### Added - Read the docs capabilities @@ -17,8 +17,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Spawn worker pools of `LvRegionCoordinator`s and `LvGridGenerator`s - Forward results to `ResultEventListener` - Coordinated shut down phase + - Post stop phase for terminated children (to shut down data connections, ...) + - Await response from terminated children ### Changed +- ... ### Fixed - ... + +[Unreleased]: https://github.com/ie3-institute/OSMoGrid/compare/7e598e53e333c9c1a7b19906584f0357ddf07990...HEAD diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index b41970b8..c581aeff 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -16,17 +16,22 @@ import edu.ie3.datamodel.models.input.container.{ import edu.ie3.datamodel.utils.ContainerUtils import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.Stopping import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, Request} import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids +import org.slf4j.Logger import java.util.UUID import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} -object OsmoGridGuardian extends RunSupport with SubGridHandling { +object OsmoGridGuardian + extends RunSupport + with StopSupport + with SubGridHandling { /* Messages, that are understood and sent */ sealed trait Request @@ -99,39 +104,71 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { def append(run: RunData): GuardianData = this.copy(runs = runs + (run.runId -> run)) + def replace(run: RunData): GuardianData = append(run) + def remove(runId: UUID): GuardianData = this.copy(runs = runs - runId) } - /** Meta data regarding a certain given generation run - * - * @param runId - * Identifier of the run - * @param cfg - * Configuration for that given run - * @param resultEventListener - * Listeners to inform about results - * @param inputDataProvider - * Reference to the input data provider - */ - private[guardian] final case class RunData( - runId: UUID, - cfg: OsmoGridConfig, - resultEventListener: Seq[ActorRef[ResultListener.ResultEvent]], - inputDataProvider: ActorRef[InputDataProvider.Request] - ) + private[guardian] sealed trait RunData { + val runId: UUID + val cfg: OsmoGridConfig + } private[guardian] case object RunData { def apply( run: Run, resultEventListener: Seq[ActorRef[ResultListener.ResultEvent]], inputDataProvider: ActorRef[InputDataProvider.Request] ): RunData = - RunData( + Running( run.runId, run.cfg, run.additionalListener ++ resultEventListener, inputDataProvider ) + + /** Meta data regarding a certain given generation run, that yet is active + * + * @param runId + * Identifier of the run + * @param cfg + * Configuration for that given run + * @param resultEventListener + * Listeners to inform about results + * @param inputDataProvider + * Reference to the input data provider + */ + private[guardian] final case class Running( + override val runId: UUID, + override val cfg: OsmoGridConfig, + resultEventListener: Seq[ActorRef[ResultListener.ResultEvent]], + inputDataProvider: ActorRef[InputDataProvider.Request] + ) extends RunData { + def toStopping: Stopping = + Stopping(runId, cfg) + } + + /** Meta data regarding a certain given generation run, that is scheduled to + * be stopped + * + * @param runId + * Identifier of the run + * @param cfg + * Configuration for that given run + * @param resultListenerTerminated + * If the result listener yet has terminated + * @param inputDataProviderTerminated + * If the input data provider yet has terminated + */ + private[guardian] final case class Stopping( + override val runId: UUID, + override val cfg: OsmoGridConfig, + resultListenerTerminated: Boolean = false, + inputDataProviderTerminated: Boolean = false + ) extends RunData { + def successfullyTerminated: Boolean = + resultListenerTerminated && inputDataProviderTerminated + } } def apply(): Behavior[Request] = Behaviors.setup { context => @@ -149,7 +186,7 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { idle(GuardianData(messageAdapters, Map.empty)) } - private def idle(guardianData: GuardianData): Behavior[Request] = + private[guardian] def idle(guardianData: GuardianData): Behavior[Request] = Behaviors.receive { case (ctx, run: Run) => initRun(run, ctx, guardianData.msgAdapters.lvCoordinator) match { @@ -176,20 +213,7 @@ object OsmoGridGuardian extends RunSupport with SubGridHandling { idle(stopRunProcesses(guardianData, runId, ctx)) } case (ctx, watch: GuardianWatch) => - ctx.log.error( - s"Received dead message '$watch' for run ${watch.runId}! " + - s"Stopping all corresponding children ..." - ) - - idle(stopRunProcesses(guardianData, watch.runId, ctx)) + /* Somebody died. Let's see, what we can do. */ + handleGuardianWatchEvent(watch, guardianData, ctx) } - - private def stopRunProcesses( - guardianData: GuardianData, - runId: UUID, - ctx: ActorContext[Request] - ): GuardianData = { - guardianData.runs.get(runId).foreach(stopChildrenByRun(_, ctx)) - guardianData.remove(runId) - } } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala index f6bc28a4..709b44df 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala @@ -182,21 +182,4 @@ trait RunSupport { ctx.log.info("Starting voltage level grid generation ...") lvCoordinator ! ReqLvGrids(runId, lvConfig, lvCoordinatorAdapter) } - - /** Stop all children for the given run - * - * @param runData - * Current run meta data - * @param ctx - * Current actor context - */ - protected def stopChildrenByRun( - runData: RunData, - ctx: ActorContext[Request] - ): Unit = { - ctx.unwatch(runData.inputDataProvider) - ctx.stop(runData.inputDataProvider) - runData.resultEventListener.foreach(ctx.unwatch) - runData.resultEventListener.foreach(ctx.stop) - } } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala new file mode 100644 index 00000000..f23fadb7 --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala @@ -0,0 +1,193 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian + +import akka.actor.typed.Behavior +import akka.actor.typed.scaladsl.{ActorContext, Behaviors} +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.Stopping +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ + GuardianData, + GuardianWatch, + InputDataProviderDied, + LvCoordinatorDied, + Request, + ResultEventListenerDied, + RunData, + idle +} +import org.slf4j.Logger + +import java.util.UUID + +trait StopSupport { + + protected def stopRunProcesses( + guardianData: GuardianData, + runId: UUID, + ctx: ActorContext[Request] + ): GuardianData = guardianData.runs + .get(runId) + .map { + case running: RunData.Running => + val stoppingRun = stopChildrenByRun(running, ctx) + guardianData.replace(stoppingRun) + case stopping: RunData.Stopping if !stopping.successfullyTerminated => + ctx.log + .debug(s"Children for run $runId already scheduled for stopping.") + guardianData + case stopping: RunData.Stopping => + ctx.log.warn(s"Children for run $runId already successfully stopped.") + guardianData + } + .getOrElse { + ctx.log.debug(s"Cannot stop run children. No run with id $runId found.") + guardianData + } + + /** Stop all children for the given run + * + * @param runData + * Current run meta data + * @param ctx + * Current actor context + */ + private def stopChildrenByRun( + runData: RunData.Running, + ctx: ActorContext[Request] + ): RunData.Stopping = { + ctx.stop(runData.inputDataProvider) + runData.resultEventListener.foreach(ctx.stop) + runData.toStopping + } + + /** Handle a [[GuardianWatch]] message and act accordingly. Either register + * successful shutdown of children in coordinated shutdown phase or initiate + * it, if somebody died unexpectedly. + * + * @param watchMsg + * Received [[GuardianWatch]] message + * @param guardianData + * Current [[GuardianData]] + * @param ctx + * Current Actor context + * @return + * Next state with updated [[GuardianData]] + */ + protected def handleGuardianWatchEvent( + watchMsg: GuardianWatch, + guardianData: GuardianData, + ctx: ActorContext[Request] + ): Behavior[Request] = { + implicit val log: Logger = ctx.log + guardianData.runs.get(watchMsg.runId) match { + case Some(stopping: RunData.Stopping) => + /* This run is scheduled for stopping. Register the replies. */ + registerCoordinatedShutDown(watchMsg, stopping, guardianData) + case Some(running: RunData.Running) => + /* This run is NOT scheduled for shutdown. Start coordinated shut down phase. */ + handleUnexpectedShutDown(watchMsg, guardianData, ctx) + case None => + ctx.log.warn( + s"Received a watch message '$watchMsg' for a run, that is not known to the guardian. Just keep on going..." + ) + Behaviors.same + } + } + + /** Register [[GuardianWatch]] messages within the coordinated shutdown phase + * of a run + * + * @param watchMsg + * Received [[GuardianWatch]] message + * @param stopping + * State data for the stopping run + * @param guardianData + * Current [[GuardianData]] + * @param log + * Logger + * @return + * Next state with updated [[GuardianData]] + */ + private def registerCoordinatedShutDown( + watchMsg: GuardianWatch, + stopping: RunData.Stopping, + guardianData: GuardianData + )(implicit log: Logger): Behavior[Request] = { + val updatedGuardianData = watchMsg match { + case InputDataProviderDied(_) => + updateGuardianData( + stopping.copy(inputDataProviderTerminated = true), + guardianData + ) + case ResultEventListenerDied(runId) => + updateGuardianData( + stopping.copy(resultListenerTerminated = true), + guardianData + ) + case LvCoordinatorDied(runId) => + log.debug( + s"LV coordinator died in coordinated shutdown phase for run $runId." + ) + guardianData + } + idle(updatedGuardianData) + } + + /** Update the guardian data by removing or updating the meta information + * about the given run + * + * @param runData + * Most recent run meta information + * @param guardianData + * Data to update + * @param log + * Logger + * @return + * The updated [[GuardianData]] + */ + private def updateGuardianData( + runData: Stopping, + guardianData: GuardianData + )(implicit log: Logger): GuardianData = if (runData.successfullyTerminated) { + log.debug(s"Run with id ${runData.runId} successfully terminated.") + guardianData.remove(runData.runId) + } else guardianData.replace(runData) + + /** Handle an unexpected shutdown of children and start coordinated shutdown + * phase for that run + * + * @param watchMsg + * Received [[GuardianWatch]] message + * @param guardianData + * Current [[GuardianData]] + * @param ctx + * Current Actor context + * @return + * Next state with updated [[GuardianData]] + */ + private def handleUnexpectedShutDown( + watchMsg: GuardianWatch, + guardianData: GuardianData, + ctx: ActorContext[Request] + ) = { + watchMsg match { + case InputDataProviderDied(runId) => + ctx.log.warn( + s"Input data provider for run $runId unexpectedly died. Start coordinated shut down phase for this run." + ) + case ResultEventListenerDied(runId) => + ctx.log.warn( + s"One of the result listener for run $runId unexpectedly died. Start coordinated shut down phase for this run." + ) + case LvCoordinatorDied(runId) => + ctx.log.warn( + s"Lv coordinator for run $runId unexpectedly died. Start coordinated shut down phase for this run." + ) + } + idle(stopRunProcesses(guardianData, watchMsg.runId, ctx)) + } +} diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala index 2a099649..6880389d 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala @@ -9,6 +9,7 @@ package edu.ie3.osmogrid.guardian import akka.actor.typed.scaladsl.ActorContext import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.datamodel.utils.ContainerUtils +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.Running import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ GuardianData, Request, @@ -43,7 +44,7 @@ trait SubGridHandling { guardianData.runs.get(runId) match { case Some( - RunData(runId, cfg, resultEventListener, inputDataProvider) + RunData.Running(runId, cfg, resultEventListener, inputDataProvider) ) => // TODO: Check for mv config and issue run there, if applicable ctx.log.debug( From b6b55bff553896851d25250d6b6912015396ba53 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 09:50:21 +0100 Subject: [PATCH 13/38] Only terminate internal result listener --- CHANGELOG.md | 1 + .../osmogrid/guardian/OsmoGridGuardian.scala | 25 +++++++++++++++---- .../ie3/osmogrid/guardian/RunSupport.scala | 9 ++++--- .../ie3/osmogrid/guardian/StopSupport.scala | 5 ++-- .../osmogrid/guardian/SubGridHandling.scala | 4 +-- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff3cc068..1ee42da9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Spawn worker pools of `LvRegionCoordinator`s and `LvGridGenerator`s - Forward results to `ResultEventListener` - Coordinated shut down phase + - Only terminate OSMoGrid internal result event listener and let additional listeners alive - Post stop phase for terminated children (to shut down data connections, ...) - Await response from terminated children diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index c581aeff..479c6cc1 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -117,13 +117,16 @@ object OsmoGridGuardian private[guardian] case object RunData { def apply( run: Run, - resultEventListener: Seq[ActorRef[ResultListener.ResultEvent]], + osmoGridResultEventListener: Option[ActorRef[ + ResultListener.ResultEvent + ]], inputDataProvider: ActorRef[InputDataProvider.Request] ): RunData = Running( run.runId, run.cfg, - run.additionalListener ++ resultEventListener, + osmoGridResultEventListener, + run.additionalListener, inputDataProvider ) @@ -133,19 +136,31 @@ object OsmoGridGuardian * Identifier of the run * @param cfg * Configuration for that given run - * @param resultEventListener - * Listeners to inform about results + * @param osmoGridResultEventListener + * Reference to internal [[ResultListener]] + * @param additionalResultListener + * References to additional [[ResultListener]] * @param inputDataProvider * Reference to the input data provider */ private[guardian] final case class Running( override val runId: UUID, override val cfg: OsmoGridConfig, - resultEventListener: Seq[ActorRef[ResultListener.ResultEvent]], + osmoGridResultEventListener: Option[ActorRef[ + ResultListener.ResultEvent + ]], + private val additionalResultListener: Seq[ + ActorRef[ResultListener.ResultEvent] + ], inputDataProvider: ActorRef[InputDataProvider.Request] ) extends RunData { def toStopping: Stopping = Stopping(runId, cfg) + + def resultListener: Seq[ActorRef[ResultListener.ResultEvent]] = + osmoGridResultEventListener + .map(Seq(_)) + .getOrElse(Seq.empty) ++ additionalResultListener } /** Meta data regarding a certain given generation run, that is scheduled to diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala index 709b44df..d5f5eb6b 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala @@ -91,6 +91,9 @@ trait RunSupport { inputConfig: OsmoGridConfig.Input, outputConfig: OsmoGridConfig.Output, ctx: ActorContext[Request] + ): ( + ActorRef[InputDataProvider.Request], + Option[ActorRef[ResultListener.ResultEvent]] ) = ( spawnInputDataProvider(runId, inputConfig, ctx), spawnResultListener(runId, outputConfig, ctx) @@ -137,11 +140,11 @@ trait RunSupport { runId: UUID, outputConfig: OsmoGridConfig.Output, ctx: ActorContext[Request] - ): Seq[ActorRef[ResultListener.ResultEvent]] = { + ): Option[ActorRef[ResultListener.ResultEvent]] = { val resultListener = outputConfig match { case Output(Some(_)) => ctx.log.info("Starting output data listener ...") - Seq( + Some( ctx.spawn( ResultListener(runId, outputConfig), s"PersistenceResultListener_${runId.toString}" @@ -149,7 +152,7 @@ trait RunSupport { ) case Output(None) => ctx.log.warn(s"No result listener configured for run $runId.") - Seq.empty + None } resultListener.foreach( ctx.watchWith(_, ResultEventListenerDied(runId)) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala index f23fadb7..71608795 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala @@ -48,7 +48,8 @@ trait StopSupport { guardianData } - /** Stop all children for the given run + /** Stop all children for the given run. The additional listeners are not + * asked to be stopped! * * @param runData * Current run meta data @@ -60,7 +61,7 @@ trait StopSupport { ctx: ActorContext[Request] ): RunData.Stopping = { ctx.stop(runData.inputDataProvider) - runData.resultEventListener.foreach(ctx.stop) + runData.osmoGridResultEventListener.foreach(ctx.stop) runData.toStopping } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala index 6880389d..6b9f269a 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala @@ -44,7 +44,7 @@ trait SubGridHandling { guardianData.runs.get(runId) match { case Some( - RunData.Running(runId, cfg, resultEventListener, inputDataProvider) + runData @ RunData.Running(runId, cfg, _, _, inputDataProvider) ) => // TODO: Check for mv config and issue run there, if applicable ctx.log.debug( @@ -53,7 +53,7 @@ trait SubGridHandling { /* Bundle grid result and inform interested listeners */ val jointGrid = ContainerUtils.combineToJointGrid(updatedSubGrids.asJava) - resultEventListener.foreach { listener => + runData.resultListener.foreach { listener => listener ! ResultListener.GridResult( jointGrid, guardianData.msgAdapters.resultListener From f9d7b7f87e95c93142a50d5e8da0deb22c8bccab Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 10:03:22 +0100 Subject: [PATCH 14/38] Fix failing test --- .../edu/ie3/osmogrid/cfg/ConfigFailFast.scala | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala index 51f77745..986d61d5 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala @@ -77,10 +77,8 @@ object ConfigFailFast extends LazyLogging { file match { case File(directory, _) if directory.isEmpty => throw IllegalConfigException("Asset input directory may be set!") - case unsupported => - throw IllegalConfigException( - s"Unable to run sanity checks on unsupported asset input file config '$unsupported'." - ) + case _: File => + /* Everything is fine */ } private def checkOsmInputConfig(osm: OsmoGridConfig.Input.Osm): Unit = @@ -96,10 +94,8 @@ object ConfigFailFast extends LazyLogging { pbf match { case OsmoGridConfig.Input.Osm.Pbf(file) if file.isEmpty => throw IllegalConfigException("Pbf file may be set!") - case unsupported => - throw IllegalConfigException( - s"Unable to run sanity checks on unsupported input file config '$unsupported'." - ) + case OsmoGridConfig.Input.Osm.Pbf(_) => + /* Everything is fine */ } private def checkOutputConfig( @@ -126,9 +122,7 @@ object ConfigFailFast extends LazyLogging { throw IllegalConfigException( "Output directory and separator must be set when using .csv file sink!" ) - case unsupported => - throw IllegalConfigException( - s"Unable to run sanity checks on unsupported output file config '$unsupported'." - ) + case _: OsmoGridConfig.Output.Csv => + /* Everything is fine */ } } From 8a34512d64d25272662cbfc638cd1aba8413ddac Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 10:10:04 +0100 Subject: [PATCH 15/38] Ignore incoming results for runs in shutdown phase --- .../scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala index 6b9f269a..4890c33f 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala @@ -9,7 +9,7 @@ package edu.ie3.osmogrid.guardian import akka.actor.typed.scaladsl.ActorContext import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.datamodel.utils.ContainerUtils -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.Running +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.{Running, Stopping} import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ GuardianData, Request, @@ -59,6 +59,10 @@ trait SubGridHandling { guardianData.msgAdapters.resultListener ) } + case Some(stopping: Stopping) => + ctx.log.warn( + s"Received results for run $runId, which is in coordinated shutdown phase. Ignore the results" + ) case None => ctx.log.error( s"Cannot find run information for '$runId', although it is supposed to be active. No further actions taken." From e642196245a1dbffec8413dc55fa7e40ae099df0 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 12:35:46 +0100 Subject: [PATCH 16/38] Add test for sub grid handling --- build.gradle | 4 + .../osmogrid/guardian/OsmoGridGuardian.scala | 2 +- .../osmogrid/guardian/SubGridHandling.scala | 19 +- .../guardian/SubGridHandlingSpec.scala | 171 ++++++++++++++++++ .../edu/ie3/test/common/GridSupport.scala | 87 +++++++++ 5 files changed, 273 insertions(+), 10 deletions(-) create mode 100644 src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala create mode 100644 src/test/scala/edu/ie3/test/common/GridSupport.scala diff --git a/build.gradle b/build.gradle index 7f4f168d..98d6e2e9 100644 --- a/build.gradle +++ b/build.gradle @@ -60,6 +60,8 @@ dependencies { // akka implementation "com.typesafe.akka:akka-actor-typed_${scalaMajorVersion}:${akkaVersion}" + implementation platform("com.typesafe.akka:akka-bom_$scalaMajorVersion:2.6.18") + testImplementation "com.typesafe.akka:akka-actor-testkit-typed_$scalaMajorVersion:2.6.18" // logging implementation "com.typesafe.scala-logging:scala-logging_${scalaMajorVersion}:3.9.4" @@ -105,12 +107,14 @@ dependencies { implementation 'math.geom2d:javaGeom:0.11.1' implementation 'info.picocli:picocli:4.6.2' // command line interface + spotbugsPlugins 'com.h3xstream.findsecbugs:findsecbugs-plugin:1.11.0' spotbugsPlugins 'com.h3xstream.findsecbugs:findsecbugs-plugin:1.11.0' // testing testImplementation 'junit:junit:4.13.2' testImplementation 'cglib:cglib-nodep:3.3.0' // enables mocking of classes (in addition to interfaces) testImplementation 'org.mockito:mockito-core:4.2.0' // mocking framework + testImplementation 'org.scalatestplus:mockito-3-12_3:3.2.10.0' // syntactic sugar testImplementation 'org.spockframework:spock-core:2.0-groovy-3.0' testImplementation 'org.objenesis:objenesis:3.2' // Mock creation with constructor parameters diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 479c6cc1..8966471a 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -216,7 +216,7 @@ object OsmoGridGuardian case (ctx, MessageAdapters.WrappedLvCoordinatorResponse(response)) => response match { case LvCoordinator.RepLvGrids(runId, grids) => - handleLvResults(runId, grids, guardianData, ctx) + handleLvResults(runId, grids, guardianData)(ctx.log) Behaviors.same } case (ctx, MessageAdapters.WrappedListenerResponse(response)) => diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala index 4890c33f..0518f036 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala @@ -15,7 +15,9 @@ import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ Request, RunData } +import edu.ie3.osmogrid.guardian.SubGridHandling.assignSubnetNumbers import edu.ie3.osmogrid.io.output.ResultListener +import org.slf4j.Logger import java.util.UUID import scala.jdk.CollectionConverters.* @@ -30,16 +32,13 @@ trait SubGridHandling { * Received grids * @param guardianData * Relevant, state-independent data, the the actor needs to know - * @param ctx - * Current actor context */ protected def handleLvResults( runId: UUID, grids: Seq[SubGridContainer], - guardianData: GuardianData, - ctx: ActorContext[Request] - ): Unit = { - ctx.log.info("All lv grids successfully generated.") + guardianData: GuardianData + )(implicit log: Logger): Unit = { + log.info("All lv grids successfully generated.") val updatedSubGrids = assignSubnetNumbers(grids) guardianData.runs.get(runId) match { @@ -47,7 +46,7 @@ trait SubGridHandling { runData @ RunData.Running(runId, cfg, _, _, inputDataProvider) ) => // TODO: Check for mv config and issue run there, if applicable - ctx.log.debug( + log.debug( "No further generation steps intended. Hand over results to result handler." ) /* Bundle grid result and inform interested listeners */ @@ -60,16 +59,18 @@ trait SubGridHandling { ) } case Some(stopping: Stopping) => - ctx.log.warn( + log.warn( s"Received results for run $runId, which is in coordinated shutdown phase. Ignore the results" ) case None => - ctx.log.error( + log.error( s"Cannot find run information for '$runId', although it is supposed to be active. No further actions taken." ) } } +} +object SubGridHandling { private def assignSubnetNumbers( subnets: Seq[SubGridContainer] ): Seq[SubGridContainer] = subnets.zipWithIndex.map { diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala new file mode 100644 index 00000000..d7d65093 --- /dev/null +++ b/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala @@ -0,0 +1,171 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian + +import akka.actor.testkit.typed.Effect.MessageAdapter +import akka.actor.testkit.typed.scaladsl.{ActorTestKit, TestProbe} +import edu.ie3.datamodel.models.input.container.SubGridContainer +import edu.ie3.osmogrid.cfg.{OsmoGridConfig, OsmoGridConfigFactory} +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ + GuardianData, + MessageAdapters, + RunData +} +import edu.ie3.osmogrid.guardian.SubGridHandling +import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener +import edu.ie3.osmogrid.lv.LvCoordinator +import edu.ie3.test.common.{GridSupport, UnitSpec} +import org.influxdb.annotation.Measurement +import org.mockito.Mockito.when +import org.scalatest.BeforeAndAfterAll +import org.scalatest.PrivateMethodTester.PrivateMethod +import org.scalatestplus.mockito.MockitoSugar.mock +import org.slf4j.{Logger, LoggerFactory} + +import java.util.UUID +import scala.jdk.CollectionConverters.* + +class SubGridHandlingSpec + extends UnitSpec + with GridSupport + with BeforeAndAfterAll { + private val testKit: ActorTestKit = ActorTestKit() + + "Supporting sub grid handling" when { + "assigning sub grid numbers to a single sub grid container" should { + val assignSubnetNumber = + PrivateMethod[SubGridContainer](Symbol("assignSubnetNumber")) + + "return the same container" in { + /* ATTENTION: This is a dummy test until the concrete logic is implemented */ + val subGridContainer = mock[SubGridContainer] + + val actual = + SubGridHandling invokePrivate assignSubnetNumber(subGridContainer, 42) + + actual shouldBe subGridContainer + } + } + + "assigning sub grid numbers to a series of sub grid containers" should { + val assignSubnetNumbers = + PrivateMethod[Seq[SubGridContainer]](Symbol("assignSubnetNumbers")) + + "return the same containers" in { + /* ATTENTION: This is a dummy test until the concrete logic is implemented */ + val containers = Range(1, 10).map(_ => mock[SubGridContainer]) + + val actual = + SubGridHandling invokePrivate assignSubnetNumbers(containers) + + actual should contain theSameElementsAs containers + } + } + + "handling incoming results" when { + implicit val log: Logger = + LoggerFactory.getLogger("SubGridHandlingTestLogger") + + val inputDataProvider = + testKit.createTestProbe[InputDataProvider.Request]("InputDataProvider") + val lvCoordinatorAdapter = + testKit.createTestProbe[LvCoordinator.Response]("LvCoordinatorAdapter") + val resultListener = + testKit.createTestProbe[ResultListener.ResultEvent]("ResultListener") + val resultListenerAdapter = + testKit.createTestProbe[ResultListener.Response]( + "ResultListenerAdapter" + ) + val additionalResultListener = + testKit.createTestProbe[ResultListener.ResultEvent]( + "AdditionalResultListener" + ) + + val runId = UUID.randomUUID() + val grids = Range(1, 10).map(mockSubGrid) + val messageAdapters = new OsmoGridGuardian.MessageAdapters( + lvCoordinatorAdapter.ref, + resultListenerAdapter.ref + ) + val cfg = OsmoGridConfigFactory.parse { + """ + |input.osm.file.pbf=test.pbf + |input.asset.file.directory=assets/ + |output.csv.directory=output/ + |generation.lv.distinctHouseConnections=true""".stripMargin + }.get + + "having an active run" should { + val guardianData = GuardianData( + messageAdapters, + Map( + runId -> RunData.Running( + runId, + cfg, + Some(resultListener.ref), + Seq(additionalResultListener.ref), + inputDataProvider.ref + ) + ) + ) + "inform the right parties about correct information" in new SubGridHandling { + handleLvResults(runId, grids, guardianData) + + resultListener.receiveMessage() match { + case ResultListener.GridResult(grid, _) => + grid.getGridName shouldBe "DummyGrid" + grid.getRawGrid.getNodes.size() shouldBe 0 + } + additionalResultListener.receiveMessage() match { + case ResultListener.GridResult(grid, _) => + grid.getGridName shouldBe "DummyGrid" + grid.getRawGrid.getNodes.size() shouldBe 0 + } + inputDataProvider.expectNoMessage() + lvCoordinatorAdapter.expectNoMessage() + resultListenerAdapter.expectNoMessage() + } + } + + "having a run in coordinated shutdown phase" should { + val guardianData = GuardianData( + messageAdapters, + Map( + runId -> RunData.Stopping( + runId, + cfg + ) + ) + ) + "inform nobody about anything" in new SubGridHandling { + resultListener.expectNoMessage() + additionalResultListener.expectNoMessage() + inputDataProvider.expectNoMessage() + lvCoordinatorAdapter.expectNoMessage() + resultListenerAdapter.expectNoMessage() + } + } + + "having no matching run" should { + val guardianData = GuardianData( + messageAdapters, + Map.empty[UUID, RunData] + ) + "inform nobody about anything" in new SubGridHandling { + resultListener.expectNoMessage() + additionalResultListener.expectNoMessage() + inputDataProvider.expectNoMessage() + lvCoordinatorAdapter.expectNoMessage() + resultListenerAdapter.expectNoMessage() + } + } + } + } + + override protected def afterAll(): Unit = testKit.shutdownTestKit() +} diff --git a/src/test/scala/edu/ie3/test/common/GridSupport.scala b/src/test/scala/edu/ie3/test/common/GridSupport.scala new file mode 100644 index 00000000..00317988 --- /dev/null +++ b/src/test/scala/edu/ie3/test/common/GridSupport.scala @@ -0,0 +1,87 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.test.common + +import edu.ie3.datamodel.models.input.{MeasurementUnitInput, NodeInput} +import edu.ie3.datamodel.models.input.connector.{ + LineInput, + SwitchInput, + Transformer2WInput, + Transformer3WInput +} +import edu.ie3.datamodel.models.input.container.{ + GraphicElements, + RawGridElements, + SubGridContainer, + SystemParticipants +} +import edu.ie3.datamodel.models.input.graphics.{ + LineGraphicInput, + NodeGraphicInput +} +import edu.ie3.datamodel.models.input.system.{ + BmInput, + ChpInput, + EvInput, + EvcsInput, + FixedFeedInInput, + HpInput, + LoadInput, + PvInput, + StorageInput, + WecInput +} +import org.mockito.Mockito.when +import org.mockito.stubbing.OngoingStubbing +import org.scalatestplus.mockito.MockitoSugar.mock + +import scala.jdk.CollectionConverters._ + +trait GridSupport { + protected def mockSubGrid(subnetNo: Int): SubGridContainer = { + val mockedRawGrid = mock[RawGridElements] + when(mockedRawGrid.getNodes).thenReturn(Set.empty[NodeInput].asJava) + when(mockedRawGrid.getLines).thenReturn(Set.empty[LineInput].asJava) + when(mockedRawGrid.getTransformer2Ws).thenReturn( + Set.empty[Transformer2WInput].asJava + ) + when(mockedRawGrid.getTransformer3Ws).thenReturn( + Set.empty[Transformer3WInput].asJava + ) + when(mockedRawGrid.getSwitches).thenReturn(Set.empty[SwitchInput].asJava) + when(mockedRawGrid.getMeasurementUnits).thenReturn( + Set.empty[MeasurementUnitInput].asJava + ) + + val mockedParticipants = new SystemParticipants( + Set.empty[BmInput].asJava, + Set.empty[ChpInput].asJava, + Set.empty[EvcsInput].asJava, + Set.empty[EvInput].asJava, + Set.empty[FixedFeedInInput].asJava, + Set.empty[HpInput].asJava, + Set.empty[LoadInput].asJava, + Set.empty[PvInput].asJava, + Set.empty[StorageInput].asJava, + Set.empty[WecInput].asJava + ) + + val mockedGraphics = new GraphicElements( + Set.empty[NodeGraphicInput].asJava, + Set.empty[LineGraphicInput].asJava + ) + + val mockedSubGrid = mock[SubGridContainer] + when(mockedSubGrid.getGridName).thenReturn(s"DummyGrid") + when(mockedSubGrid.getSubnet).thenReturn(subnetNo) + when(mockedSubGrid.getRawGrid).thenReturn(mockedRawGrid) + when(mockedSubGrid.getSystemParticipants).thenReturn(mockedParticipants) + when(mockedSubGrid.getGraphics).thenReturn(mockedGraphics) + + mockedSubGrid + } +} From a6e696e8357f2cfcd9bbd42127bc157adff7fa53 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 12:50:32 +0100 Subject: [PATCH 17/38] Remove last Vectors --- src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala | 5 ++--- .../scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala | 3 +-- src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala index 986d61d5..949667eb 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala @@ -19,8 +19,7 @@ import edu.ie3.osmogrid.io.output.ResultListener object ConfigFailFast extends LazyLogging { def check( cfg: OsmoGridConfig, - additionalListener: Vector[ActorRef[ResultListener.ResultEvent]] = - Vector.empty + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]] = Seq.empty ): Unit = cfg match { case OsmoGridConfig(generation, input, output) => checkInputConfig(input) @@ -100,7 +99,7 @@ object ConfigFailFast extends LazyLogging { private def checkOutputConfig( output: OsmoGridConfig.Output, - additionalListener: Vector[ActorRef[ResultListener.ResultEvent]] + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]] ): Unit = output match { case Output(Some(file)) => diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 8966471a..4dd58426 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -47,8 +47,7 @@ object OsmoGridGuardian */ final case class Run( cfg: OsmoGridConfig, - additionalListener: Vector[ActorRef[ResultListener.ResultEvent]] = - Vector.empty, + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]] = Seq.empty, runId: UUID = UUID.randomUUID() ) extends Request diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala index cb658fd6..a63fd137 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala @@ -68,7 +68,7 @@ object LvCoordinator { val lvRegionCoordinatorProxy = ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") - replyTo ! RepLvGrids(runId, Vector.empty[SubGridContainer]) + replyTo ! RepLvGrids(runId, Seq.empty[SubGridContainer]) Behaviors.stopped case unsupported => ctx.log.error(s"Received unsupported message: $unsupported") From 362ad3a508ab1fa7b7e9ef1577ec333db15576f1 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 12:52:37 +0100 Subject: [PATCH 18/38] Remove unused import --- .../scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala index d7d65093..dd671455 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala @@ -20,7 +20,6 @@ import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.test.common.{GridSupport, UnitSpec} -import org.influxdb.annotation.Measurement import org.mockito.Mockito.when import org.scalatest.BeforeAndAfterAll import org.scalatest.PrivateMethodTester.PrivateMethod From 933f6e55e490e04b0717eac58211ea9540fd27f7 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 14:20:11 +0100 Subject: [PATCH 19/38] Improve protocol --- docs/puml/protocol/protocol.puml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/puml/protocol/protocol.puml b/docs/puml/protocol/protocol.puml index 63898ac0..c321ca8c 100644 --- a/docs/puml/protocol/protocol.puml +++ b/docs/puml/protocol/protocol.puml @@ -8,6 +8,7 @@ boundary Main database Input participant InputDataProvider participant OsmoGridGuardian +participant RunGuardian participant LvCoordinator participant LvRegionCoordinator participant MunicipalityCoordinator @@ -17,15 +18,17 @@ participant LvGridGenerator participant ResultListener == Init == -Main -> OsmoGridGuardian: Run(cfg) -OsmoGridGuardian --> InputDataProvider: //Spawn// +Main -> OsmoGridGuardian: !Run(cfg, ...) +OsmoGridGuardian --> RunGuardian: //Spawn// +OsmoGridGuardian -> RunGuardian: !Run(cfg, ...) +RunGuardian --> InputDataProvider: //Spawn// -OsmoGridGuardian --> ResultListener: //Spawn// +RunGuardian --> ResultListener: //Spawn// note right: Death watch of\n""ResultListener"" == LV generation == -OsmoGridGuardian --> LvCoordinator: //Spawn// -OsmoGridGuardian -> LvCoordinator: !ReqLvGrids(...) +RunGuardian --> LvCoordinator: //Spawn// +RunGuardian -> LvCoordinator: !ReqLvGrids(...) LvCoordinator -> InputDataProvider: !ReqOsm(...) InputDataProvider <--> Input: //Read// LvCoordinator -> InputDataProvider: !ReqAssetTypes(...) @@ -68,21 +71,22 @@ LvRegionCoordinator -> LvRegionCoordinator: !Done LvRegionCoordinator -> LvCoordinator: !Done note left: All results are apparent;\nassign sub grid numbers -LvCoordinator -> OsmoGridGuardian: !RepLvGrids(...) +LvCoordinator -> RunGuardian: !RepLvGrids(...) == MV generation == ... **To be defined in a later stage** ... == Result handling == -OsmoGridGuardian -> ResultListener: !GridResult(...) +RunGuardian -> ResultListener: !GridResult(...) activate ResultListener ... ... -ResultListener -> OsmoGridGuardian: !ResultListenerDied +ResultListener -> RunGuardian: !ResultListenerDied deactivate ResultListener -OsmoGridGuardian -> InputDataProvider: !Terminate(...) +RunGuardian -> InputDataProvider: !Terminate(...) InputDataProvider <--> Input: //Close// -InputDataProvider -> OsmoGridGuardian: !InputDataProviderDied +InputDataProvider -> RunGuardian: !InputDataProviderDied +RunGuardian -> OsmoGridGuardian: !Done OsmoGridGuardian -> Main: !Done 'TODO: Don't forget to spawn and initialize the ResultListener From 6474c29c688efa3025a24c42afa317f0c3023a9a Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Tue, 18 Jan 2022 15:22:11 +0100 Subject: [PATCH 20/38] Introduction of a RunGuardian to handle single runs --- CHANGELOG.md | 12 +- .../osmogrid/guardian/OsmoGridGuardian.scala | 188 ++----------- .../ie3/osmogrid/guardian/RunGuardian.scala | 256 ++++++++++++++++++ .../ie3/osmogrid/guardian/RunSupport.scala | 77 +++--- .../ie3/osmogrid/guardian/StopSupport.scala | 186 +++---------- .../osmogrid/guardian/SubGridHandling.scala | 61 ++--- .../osmogrid/io/output/ResultListener.scala | 7 +- .../guardian/SubGridHandlingSpec.scala | 60 +--- 8 files changed, 408 insertions(+), 439 deletions(-) create mode 100644 src/main/scala/edu/ie3/osmogrid/guardian/RunGuardian.scala diff --git a/CHANGELOG.md b/CHANGELOG.md index f2321173..10b055f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Input - Output - Generation -- Let `OsmoGridGuardian` initialize services -- Spawn `LvCoordinator` and trigger it -- Spawn worker pools of `LvRegionCoordinator`s and `LvGridGenerator`s -- Forward results to `ResultEventListener` +- Let `OsmoGridGuardian` handle multiple runs and spawn children accordingly +- A `RunGuardian` takes care of a distinct simulation run and spawns all its needed services + - Spawn an `InputDataProvider` and a `ResultListener`(if need be) per run + - Spawn `LvCoordinator` and trigger it - Coordinated shut down phase - Only terminate OSMoGrid internal result event listener and let additional listeners alive - Post stop phase for terminated children (to shut down data connections, ...) @@ -27,8 +27,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed - Legacy Java code - - Jacoco gradle pluginfgc - -[Unreleased]: https://github.com/ie3-institute/OSMoGrid/compare/7e598e53e333c9c1a7b19906584f0357ddf07990...HEAD + - Jacoco gradle plugin [Unreleased]: https://github.com/ie3-institute/OSMoGrid/compare/7e598e53e333c9c1a7b19906584f0357ddf07990...HEAD diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 4dd58426..c5c0fa04 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -16,7 +16,6 @@ import edu.ie3.datamodel.models.input.container.{ import edu.ie3.datamodel.utils.ContainerUtils import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.Stopping import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, Request} @@ -28,10 +27,7 @@ import java.util.UUID import scala.jdk.CollectionConverters.* import scala.util.{Failure, Success, Try} -object OsmoGridGuardian - extends RunSupport - with StopSupport - with SubGridHandling { +object OsmoGridGuardian { /* Messages, that are understood and sent */ sealed trait Request @@ -51,183 +47,49 @@ object OsmoGridGuardian runId: UUID = UUID.randomUUID() ) extends Request - /** Container object with all available adapters for outside protocol messages - * - * @param lvCoordinator - * Adapter for messages from [[LvCoordinator]] - * @param resultListener - * Adapter for messages from [[ResultEventListener]] - */ - private[guardian] final case class MessageAdapters( - lvCoordinator: ActorRef[LvCoordinator.Response], - resultListener: ActorRef[ResultListener.Response] - ) - private object MessageAdapters { - final case class WrappedLvCoordinatorResponse( - response: LvCoordinator.Response - ) extends Request - - final case class WrappedListenerResponse( - response: ResultListener.Response - ) extends Request - } - - sealed trait Response - /* dead watch events */ - sealed trait GuardianWatch extends Request { + sealed trait Watch extends Request { val runId: UUID } - private[guardian] final case class InputDataProviderDied(runId: UUID) - extends GuardianWatch - - private[guardian] final case class ResultEventListenerDied(runId: UUID) - extends GuardianWatch - - private[guardian] final case class LvCoordinatorDied(runId: UUID) - extends GuardianWatch + private[guardian] final case class RunGuardianDied(override val runId: UUID) + extends Watch /** Relevant, state-independent data, the the actor needs to know * - * @param msgAdapters - * Collection of all message adapters * @param runs * Currently active conversion runs */ private[guardian] final case class GuardianData( - msgAdapters: MessageAdapters, - runs: Map[UUID, RunData] + runs: Seq[UUID] ) { + def append(run: UUID): GuardianData = this.copy(runs = runs :+ run) - def append(run: RunData): GuardianData = - this.copy(runs = runs + (run.runId -> run)) - - def replace(run: RunData): GuardianData = append(run) - - def remove(runId: UUID): GuardianData = - this.copy(runs = runs - runId) - } - - private[guardian] sealed trait RunData { - val runId: UUID - val cfg: OsmoGridConfig + def remove(run: UUID): GuardianData = + this.copy(runs = runs.filterNot(_ == run)) } - private[guardian] case object RunData { - def apply( - run: Run, - osmoGridResultEventListener: Option[ActorRef[ - ResultListener.ResultEvent - ]], - inputDataProvider: ActorRef[InputDataProvider.Request] - ): RunData = - Running( - run.runId, - run.cfg, - osmoGridResultEventListener, - run.additionalListener, - inputDataProvider - ) - - /** Meta data regarding a certain given generation run, that yet is active - * - * @param runId - * Identifier of the run - * @param cfg - * Configuration for that given run - * @param osmoGridResultEventListener - * Reference to internal [[ResultListener]] - * @param additionalResultListener - * References to additional [[ResultListener]] - * @param inputDataProvider - * Reference to the input data provider - */ - private[guardian] final case class Running( - override val runId: UUID, - override val cfg: OsmoGridConfig, - osmoGridResultEventListener: Option[ActorRef[ - ResultListener.ResultEvent - ]], - private val additionalResultListener: Seq[ - ActorRef[ResultListener.ResultEvent] - ], - inputDataProvider: ActorRef[InputDataProvider.Request] - ) extends RunData { - def toStopping: Stopping = - Stopping(runId, cfg) - - def resultListener: Seq[ActorRef[ResultListener.ResultEvent]] = - osmoGridResultEventListener - .map(Seq(_)) - .getOrElse(Seq.empty) ++ additionalResultListener - } - - /** Meta data regarding a certain given generation run, that is scheduled to - * be stopped - * - * @param runId - * Identifier of the run - * @param cfg - * Configuration for that given run - * @param resultListenerTerminated - * If the result listener yet has terminated - * @param inputDataProviderTerminated - * If the input data provider yet has terminated - */ - private[guardian] final case class Stopping( - override val runId: UUID, - override val cfg: OsmoGridConfig, - resultListenerTerminated: Boolean = false, - inputDataProviderTerminated: Boolean = false - ) extends RunData { - def successfullyTerminated: Boolean = - resultListenerTerminated && inputDataProviderTerminated - } + object GuardianData { + def empty = new GuardianData(Seq.empty[UUID]) } - def apply(): Behavior[Request] = Behaviors.setup { context => - /* Define and register message adapters */ - val messageAdapters = - MessageAdapters( - context.messageAdapter(msg => - MessageAdapters.WrappedLvCoordinatorResponse(msg) - ), - context.messageAdapter(msg => - MessageAdapters.WrappedListenerResponse(msg) - ) - ) - - idle(GuardianData(messageAdapters, Map.empty)) - } + def apply(): Behavior[Request] = idle(GuardianData.empty) private[guardian] def idle(guardianData: GuardianData): Behavior[Request] = Behaviors.receive { - case (ctx, run: Run) => - initRun(run, ctx, guardianData.msgAdapters.lvCoordinator) match { - case Success(runData) => idle(guardianData.append(runData)) - case Failure(exception) => - ctx.log.error( - "Issuing of run failed. Keep on going without that run.", - exception - ) - idle(guardianData) - } - case (ctx, MessageAdapters.WrappedLvCoordinatorResponse(response)) => - response match { - case LvCoordinator.RepLvGrids(runId, grids) => - handleLvResults(runId, grids, guardianData)(ctx.log) - Behaviors.same - } - case (ctx, MessageAdapters.WrappedListenerResponse(response)) => - response match { - case ResultListener.ResultHandled(runId) => - ctx.log.info( - s"Results for run $runId handled successfully. Shutting down processes for this run." - ) - idle(stopRunProcesses(guardianData, runId, ctx)) + case (ctx, Run(cfg, additionalListener, runId)) => + val runGuardian = ctx.spawn( + RunGuardian(cfg, additionalListener, runId), + s"RunGuardian_$runId" + ) + ctx.watchWith(runGuardian, RunGuardianDied(runId)) + runGuardian ! RunGuardian.Run + idle(guardianData.append(runId)) + + case (ctx, watch: Watch) => + watch match { + case RunGuardianDied(runId) => + ctx.log.info(s"Run $runId terminated.") + idle(guardianData.remove(runId)) } - case (ctx, watch: GuardianWatch) => - /* Somebody died. Let's see, what we can do. */ - handleGuardianWatchEvent(watch, guardianData, ctx) } } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/RunGuardian.scala new file mode 100644 index 00000000..bfc3a0a1 --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/guardian/RunGuardian.scala @@ -0,0 +1,256 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian + +import akka.actor.typed.{ActorRef, Behavior} +import akka.actor.typed.scaladsl.{Behaviors, StashBuffer} +import edu.ie3.osmogrid.cfg.OsmoGridConfig +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.Request +import edu.ie3.osmogrid.guardian.RunGuardian.MessageAdapters.{ + WrappedListenerResponse, + WrappedLvCoordinatorResponse +} +import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener +import edu.ie3.osmogrid.lv.LvCoordinator + +import java.util.UUID +import scala.util.{Failure, Success} + +/** Actor to take care of a specific simulation run + */ +object RunGuardian extends RunSupport with StopSupport with SubGridHandling { + sealed trait Request + object Run extends Request + + /** Container object with all available adapters for outside protocol messages + * + * @param lvCoordinator + * Adapter for messages from [[LvCoordinator]] + * @param resultListener + * Adapter for messages from [[ResultEventListener]] + */ + private[guardian] final case class MessageAdapters( + lvCoordinator: ActorRef[LvCoordinator.Response], + resultListener: ActorRef[ResultListener.Response] + ) + private[guardian] object MessageAdapters { + final case class WrappedLvCoordinatorResponse( + response: LvCoordinator.Response + ) extends Request + + final case class WrappedListenerResponse( + response: ResultListener.Response + ) extends Request + } + + sealed trait Response + final case class Done(runId: UUID) extends Response + + sealed trait Watch extends Request + private[guardian] object InputDataProviderDied extends Watch + + private[guardian] object ResultEventListenerDied extends Watch + + private[guardian] object LvCoordinatorDied extends Watch + + final case class RunGuardianData( + runId: UUID, + cfg: OsmoGridConfig, + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], + msgAdapters: MessageAdapters + ) + + private[guardian] final case class ChildReferences( + inputDataProvider: ActorRef[InputDataProvider.Request], + resultListener: Option[ActorRef[ResultListener.ResultEvent]], + additionalResultListeners: Seq[ActorRef[ResultListener.ResultEvent]], + lvCoordinator: Option[ActorRef[LvCoordinator.Request]] + ) { + def resultListeners: Seq[ActorRef[ResultListener.ResultEvent]] = + resultListener + .map(Seq(_)) + .getOrElse(Seq.empty) ++ additionalResultListeners + } + + /** Meta data to keep track of which children already terminated during the + * coordinated shutdown phase + * + * @param inputDataProviderTerminated + * If the [[InputDataProvider]] has stopped + * @param resultListenerTerminated + * If the [[ResultListener]] has stopped + * @param lvCoordinatorTerminated + * Optional information, if the [[LvCoordinator]] has stopped + */ + private[guardian] final case class StoppingData( + inputDataProviderTerminated: Boolean, + resultListenerTerminated: Boolean, + lvCoordinatorTerminated: Option[Boolean] + ) { + def allChildrenTerminated: Boolean = + inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated + .contains(true) + } + + /** Instantiate the actor + * + * @param cfg + * Configuration for the tool + * @param additionalListener + * Addresses of additional listeners to be informed about results + * @param runId + * Unique identifier for that generation run + */ + def apply( + cfg: OsmoGridConfig, + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]] = Seq.empty, + runId: UUID + ): Behavior[Request] = Behaviors.setup { ctx => + idle( + runId, + cfg, + additionalListener, + MessageAdapters( + ctx.messageAdapter(msg => WrappedLvCoordinatorResponse(msg)), + ctx.messageAdapter(msg => WrappedListenerResponse(msg)) + ) + ) + } + + /** This actor is in idle state and waits for any kind of request + * + * @param runId + * Identifier of the current run + * @param cfg + * Configuration for this run + * @param additionalListener + * References to additional [[ResultListener]]s + * @param msgAdapters + * Collection of all message adapters + * @return + * the next state + */ + private[guardian] def idle( + runId: UUID, + cfg: OsmoGridConfig, + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], + msgAdapters: MessageAdapters + ): Behavior[Request] = + Behaviors.receive { + case (ctx, Run) => + /* Start a run */ + initRun(runId, cfg, additionalListener, msgAdapters, ctx) match { + case Success(childReferences) => + running(runId, cfg, childReferences, msgAdapters) + case Failure(exception) => + ctx.log.error(s"Unable to start run $runId.", exception) + Behaviors.stopped + } + case (ctx, notUnderstood) => + ctx.log.error( + s"Received a message, that I don't understand during idle phase of run $runId.\n\tMessage: $notUnderstood" + ) + Behaviors.same + } + + /** Behavior to indicate, that a simulation run is currently active + * @param runId + * Identifier of the run + * @param cfg + * Configuration for this run + * @param childReferences + * References to child actors + * @param msgAdapters + * Collection of message adapters + * @return + * The next state + */ + private def running( + runId: UUID, + cfg: OsmoGridConfig, + childReferences: ChildReferences, + msgAdapters: MessageAdapters + ): Behavior[Request] = Behaviors.receive { + case ( + ctx, + WrappedLvCoordinatorResponse( + LvCoordinator.RepLvGrids(_, subGridContainers) + ) + ) => + /* Handle the grid results and wait for the listener to report back */ + handleLvResults( + subGridContainers, + cfg.generation, + childReferences.resultListeners, + msgAdapters + )(ctx.log) + Behaviors.same + case ( + ctx, + WrappedListenerResponse(ResultListener.ResultHandled(_, sender)) + ) => + ctx.log.debug( + s"The listener $sender has successfully handled the result event of run $runId." + ) + if ( + childReferences.resultListener.contains( + sender + ) || childReferences.resultListener.isEmpty + ) { + /* Start coordinated shutdown */ + ctx.log.info( + s"Run $runId successfully finished. Stop all run-related processes." + ) + stopping(runId, stopChildren(childReferences, ctx)) + } else { + /* Somebody did something great, but nothing, that affects us */ + Behaviors.same + } + case (ctx, watch: Watch) => + /* Somebody died unexpectedly. Start coordinated shutdown */ + stopping( + runId, + handleUnexpectedShutDown(runId, childReferences, watch, ctx) + ) + case (ctx, notUnderstood) => + ctx.log.error( + s"Received a message, that I don't understand during active run $runId.\n\tMessage: $notUnderstood" + ) + Behaviors.same + } + + /** Behavior, that indicates, that a coordinate shutdown of the children takes + * place + * + * @param runId + * Identifier of the run + * @param stoppingData + * Information about who already has terminated + * @return + * The next state + */ + private def stopping( + runId: UUID, + stoppingData: StoppingData + ): Behavior[Request] = Behaviors.receive { + case (ctx, watch: Watch) => + val updatedStoppingData = registerCoordinatedShutDown(watch, stoppingData) + if (updatedStoppingData.allChildrenTerminated) { + ctx.log.info( + s"All child processes of run $runId successfully terminated. Finally terminating the whole run process." + ) + /* The overall guardian is automatically informed via death watch */ + Behaviors.stopped + } else stopping(runId, updatedStoppingData) + case (ctx, notUnderstood) => + ctx.log.error( + s"Received a message, that I don't understand during coordinated shutdown phase of run $runId.\n\tMessage: $notUnderstood" + ) + Behaviors.same + } +} diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala index d5f5eb6b..a1552172 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala @@ -10,64 +10,69 @@ import akka.actor.typed.ActorRef import akka.actor.typed.scaladsl.ActorContext import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ - InputDataProviderDied, - LvCoordinatorDied, - Request, - ResultEventListenerDied, - Run, - RunData -} import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids import java.util.UUID -import scala.util.{Try, Success, Failure} +import scala.util.{Failure, Success, Try} trait RunSupport { /** Initiate a generation run and return the updated run meta data * - * @param run - * Current run meta data + * @param runId + * Identifier of the run + * @param cfg + * Configuration for the run + * @param additionalListener + * References to additional listeners + * @param msgAdapters + * Collection of all message adapters * @param ctx * Current actor context - * @param lvCoordinatorAdapter - * Message adapter to understand [[LvCoordinator]] * @return * Updated run meta data */ protected def initRun( - run: Run, - ctx: ActorContext[Request], - lvCoordinatorAdapter: ActorRef[LvCoordinator.Response] - ): Try[RunData] = { + runId: UUID, + cfg: OsmoGridConfig, + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], + msgAdapters: RunGuardian.MessageAdapters, + ctx: ActorContext[RunGuardian.Request] + ): Try[RunGuardian.ChildReferences] = { val log = ctx.log - ConfigFailFast.check(run.cfg, run.additionalListener) - log.info(s"Initializing grid generation for run with id '${run.runId}'!") + ConfigFailFast.check(cfg, additionalListener) + log.info(s"Initializing grid generation for run with id '$runId'!") /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ - run.cfg.generation match { + cfg.generation match { case Generation(Some(lvConfig)) => ctx.log.info("Starting low voltage grid coordinator ...") val (inputProvider, resultEventListener) = - spawnIoActors(run.runId, run.cfg.input, run.cfg.output, ctx) - startLvGridGeneration( - run.runId, + spawnIoActors(runId, cfg.input, cfg.output, ctx) + val lvCoordinator = startLvGridGeneration( + runId, lvConfig, - lvCoordinatorAdapter, + msgAdapters.lvCoordinator, ctx ) - Success(RunData(run, resultEventListener, inputProvider)) + Success( + RunGuardian.ChildReferences( + inputProvider, + resultEventListener, + additionalListener, + Some(lvCoordinator) + ) + ) case unsupported => ctx.log.error( - s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${run.runId}'!" + s"Received unsupported grid generation config '$unsupported'. Stopping run with id '$runId'!" ) Failure( UnsupportedOperationException( - s"Unable to issue a generation run with the given parameters: '${run.cfg.generation}'" + s"Unable to issue a generation run with the given parameters: '${cfg.generation}'" ) ) } @@ -90,7 +95,7 @@ trait RunSupport { runId: UUID, inputConfig: OsmoGridConfig.Input, outputConfig: OsmoGridConfig.Output, - ctx: ActorContext[Request] + ctx: ActorContext[RunGuardian.Request] ): ( ActorRef[InputDataProvider.Request], Option[ActorRef[ResultListener.ResultEvent]] @@ -113,7 +118,7 @@ trait RunSupport { private def spawnInputDataProvider( runId: UUID, inputConfig: OsmoGridConfig.Input, - ctx: ActorContext[Request] + ctx: ActorContext[RunGuardian.Request] ): ActorRef[InputDataProvider.Request] = { ctx.log.info("Starting input data provider ...") val inputProvider = @@ -121,7 +126,7 @@ trait RunSupport { InputDataProvider(inputConfig), s"InputDataProvider_${runId.toString}" ) - ctx.watchWith(inputProvider, InputDataProviderDied(runId)) + ctx.watchWith(inputProvider, RunGuardian.InputDataProviderDied) inputProvider } @@ -139,7 +144,7 @@ trait RunSupport { private def spawnResultListener( runId: UUID, outputConfig: OsmoGridConfig.Output, - ctx: ActorContext[Request] + ctx: ActorContext[RunGuardian.Request] ): Option[ActorRef[ResultListener.ResultEvent]] = { val resultListener = outputConfig match { case Output(Some(_)) => @@ -155,7 +160,7 @@ trait RunSupport { None } resultListener.foreach( - ctx.watchWith(_, ResultEventListenerDied(runId)) + ctx.watchWith(_, RunGuardian.ResultEventListenerDied) ) resultListener } @@ -176,13 +181,15 @@ trait RunSupport { runId: UUID, lvConfig: OsmoGridConfig.Generation.Lv, lvCoordinatorAdapter: ActorRef[LvCoordinator.Response], - ctx: ActorContext[Request] - ): Unit = { + ctx: ActorContext[RunGuardian.Request] + ): ActorRef[LvCoordinator.Request] = { val lvCoordinator = ctx.spawn(LvCoordinator(), s"LvCoordinator_${runId.toString}") - ctx.watchWith(lvCoordinator, LvCoordinatorDied(runId)) + ctx.watchWith(lvCoordinator, RunGuardian.LvCoordinatorDied) ctx.log.info("Starting voltage level grid generation ...") lvCoordinator ! ReqLvGrids(runId, lvConfig, lvCoordinatorAdapter) + + lvCoordinator } } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala index 71608795..4140e980 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala @@ -7,188 +7,92 @@ package edu.ie3.osmogrid.guardian import akka.actor.typed.Behavior -import akka.actor.typed.scaladsl.{ActorContext, Behaviors} -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.Stopping -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ - GuardianData, - GuardianWatch, - InputDataProviderDied, - LvCoordinatorDied, - Request, - ResultEventListenerDied, - RunData, - idle -} +import akka.actor.typed.scaladsl.ActorContext +import edu.ie3.osmogrid.guardian.RunGuardian.{ChildReferences, StoppingData} import org.slf4j.Logger import java.util.UUID trait StopSupport { - protected def stopRunProcesses( - guardianData: GuardianData, - runId: UUID, - ctx: ActorContext[Request] - ): GuardianData = guardianData.runs - .get(runId) - .map { - case running: RunData.Running => - val stoppingRun = stopChildrenByRun(running, ctx) - guardianData.replace(stoppingRun) - case stopping: RunData.Stopping if !stopping.successfullyTerminated => - ctx.log - .debug(s"Children for run $runId already scheduled for stopping.") - guardianData - case stopping: RunData.Stopping => - ctx.log.warn(s"Children for run $runId already successfully stopped.") - guardianData - } - .getOrElse { - ctx.log.debug(s"Cannot stop run children. No run with id $runId found.") - guardianData - } - /** Stop all children for the given run. The additional listeners are not * asked to be stopped! * - * @param runData - * Current run meta data + * @param childReferences + * References to children * @param ctx * Current actor context */ - private def stopChildrenByRun( - runData: RunData.Running, - ctx: ActorContext[Request] - ): RunData.Stopping = { - ctx.stop(runData.inputDataProvider) - runData.osmoGridResultEventListener.foreach(ctx.stop) - runData.toStopping - } + protected def stopChildren( + childReferences: RunGuardian.ChildReferences, + ctx: ActorContext[RunGuardian.Request] + ): StoppingData = { + ctx.stop(childReferences.inputDataProvider) + childReferences.resultListener.foreach(ctx.stop) - /** Handle a [[GuardianWatch]] message and act accordingly. Either register - * successful shutdown of children in coordinated shutdown phase or initiate - * it, if somebody died unexpectedly. - * - * @param watchMsg - * Received [[GuardianWatch]] message - * @param guardianData - * Current [[GuardianData]] - * @param ctx - * Current Actor context - * @return - * Next state with updated [[GuardianData]] - */ - protected def handleGuardianWatchEvent( - watchMsg: GuardianWatch, - guardianData: GuardianData, - ctx: ActorContext[Request] - ): Behavior[Request] = { - implicit val log: Logger = ctx.log - guardianData.runs.get(watchMsg.runId) match { - case Some(stopping: RunData.Stopping) => - /* This run is scheduled for stopping. Register the replies. */ - registerCoordinatedShutDown(watchMsg, stopping, guardianData) - case Some(running: RunData.Running) => - /* This run is NOT scheduled for shutdown. Start coordinated shut down phase. */ - handleUnexpectedShutDown(watchMsg, guardianData, ctx) - case None => - ctx.log.warn( - s"Received a watch message '$watchMsg' for a run, that is not known to the guardian. Just keep on going..." - ) - Behaviors.same - } + StoppingData(false, false, childReferences.lvCoordinator.map(_ => false)) } - /** Register [[GuardianWatch]] messages within the coordinated shutdown phase - * of a run + /** Register [[Watch]] messages within the coordinated shutdown phase of a run * * @param watchMsg - * Received [[GuardianWatch]] message - * @param stopping + * Received [[Watch]] message + * @param stoppingData * State data for the stopping run - * @param guardianData - * Current [[GuardianData]] - * @param log - * Logger * @return * Next state with updated [[GuardianData]] */ - private def registerCoordinatedShutDown( - watchMsg: GuardianWatch, - stopping: RunData.Stopping, - guardianData: GuardianData - )(implicit log: Logger): Behavior[Request] = { - val updatedGuardianData = watchMsg match { - case InputDataProviderDied(_) => - updateGuardianData( - stopping.copy(inputDataProviderTerminated = true), - guardianData - ) - case ResultEventListenerDied(runId) => - updateGuardianData( - stopping.copy(resultListenerTerminated = true), - guardianData - ) - case LvCoordinatorDied(runId) => - log.debug( - s"LV coordinator died in coordinated shutdown phase for run $runId." - ) - guardianData - } - idle(updatedGuardianData) + protected def registerCoordinatedShutDown( + watchMsg: RunGuardian.Watch, + stoppingData: StoppingData + ): StoppingData = watchMsg match { + case RunGuardian.InputDataProviderDied => + stoppingData.copy(inputDataProviderTerminated = true) + case RunGuardian.ResultEventListenerDied => + stoppingData.copy(resultListenerTerminated = true) + case RunGuardian.LvCoordinatorDied => + stoppingData.copy(lvCoordinatorTerminated = + stoppingData.lvCoordinatorTerminated.map(_ => true) + ) } - /** Update the guardian data by removing or updating the meta information - * about the given run - * - * @param runData - * Most recent run meta information - * @param guardianData - * Data to update - * @param log - * Logger - * @return - * The updated [[GuardianData]] - */ - private def updateGuardianData( - runData: Stopping, - guardianData: GuardianData - )(implicit log: Logger): GuardianData = if (runData.successfullyTerminated) { - log.debug(s"Run with id ${runData.runId} successfully terminated.") - guardianData.remove(runData.runId) - } else guardianData.replace(runData) - /** Handle an unexpected shutdown of children and start coordinated shutdown * phase for that run * + * @param runId + * Identifier of the current run + * @param childReferences + * References to child actors * @param watchMsg - * Received [[GuardianWatch]] message - * @param guardianData - * Current [[GuardianData]] + * Received [[Watch]] message * @param ctx * Current Actor context * @return * Next state with updated [[GuardianData]] */ - private def handleUnexpectedShutDown( - watchMsg: GuardianWatch, - guardianData: GuardianData, - ctx: ActorContext[Request] - ) = { - watchMsg match { - case InputDataProviderDied(runId) => + protected def handleUnexpectedShutDown( + runId: UUID, + childReferences: ChildReferences, + watchMsg: RunGuardian.Watch, + ctx: ActorContext[RunGuardian.Request] + ): StoppingData = { + (stopChildren(childReferences, ctx), watchMsg) match { + case (stoppingData, RunGuardian.InputDataProviderDied) => ctx.log.warn( s"Input data provider for run $runId unexpectedly died. Start coordinated shut down phase for this run." ) - case ResultEventListenerDied(runId) => + stoppingData.copy(inputDataProviderTerminated = true) + case (stoppingData, RunGuardian.ResultEventListenerDied) => ctx.log.warn( s"One of the result listener for run $runId unexpectedly died. Start coordinated shut down phase for this run." ) - case LvCoordinatorDied(runId) => + stoppingData.copy(resultListenerTerminated = true) + case (stoppingData, RunGuardian.LvCoordinatorDied) => ctx.log.warn( s"Lv coordinator for run $runId unexpectedly died. Start coordinated shut down phase for this run." ) + stoppingData.copy(resultListenerTerminated = true) } - idle(stopRunProcesses(guardianData, watchMsg.runId, ctx)) + } } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala index 0518f036..2c0bd41c 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala @@ -6,15 +6,10 @@ package edu.ie3.osmogrid.guardian -import akka.actor.typed.scaladsl.ActorContext +import akka.actor.typed.ActorRef import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.datamodel.utils.ContainerUtils -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunData.{Running, Stopping} -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ - GuardianData, - Request, - RunData -} +import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.guardian.SubGridHandling.assignSubnetNumbers import edu.ie3.osmogrid.io.output.ResultListener import org.slf4j.Logger @@ -26,46 +21,36 @@ trait SubGridHandling { /** Handle incoming low voltage grid results * - * @param runId - * Reference to the current run * @param grids * Received grids - * @param guardianData - * Relevant, state-independent data, the the actor needs to know + * @param cfg + * Grid generation config + * @param resultListener + * References to the responsible result listener + * @param msgAdapters + * Collection of all message adapters */ protected def handleLvResults( - runId: UUID, grids: Seq[SubGridContainer], - guardianData: GuardianData + cfg: OsmoGridConfig.Generation, + resultListener: Seq[ActorRef[ResultListener.ResultEvent]], + msgAdapters: RunGuardian.MessageAdapters )(implicit log: Logger): Unit = { log.info("All lv grids successfully generated.") val updatedSubGrids = assignSubnetNumbers(grids) - guardianData.runs.get(runId) match { - case Some( - runData @ RunData.Running(runId, cfg, _, _, inputDataProvider) - ) => - // TODO: Check for mv config and issue run there, if applicable - log.debug( - "No further generation steps intended. Hand over results to result handler." - ) - /* Bundle grid result and inform interested listeners */ - val jointGrid = - ContainerUtils.combineToJointGrid(updatedSubGrids.asJava) - runData.resultListener.foreach { listener => - listener ! ResultListener.GridResult( - jointGrid, - guardianData.msgAdapters.resultListener - ) - } - case Some(stopping: Stopping) => - log.warn( - s"Received results for run $runId, which is in coordinated shutdown phase. Ignore the results" - ) - case None => - log.error( - s"Cannot find run information for '$runId', although it is supposed to be active. No further actions taken." - ) + // TODO: Check for mv config and issue run there, if applicable + log.debug( + "No further generation steps intended. Hand over results to result handler." + ) + /* Bundle grid result and inform interested listeners */ + val jointGrid = + ContainerUtils.combineToJointGrid(updatedSubGrids.asJava) + resultListener.foreach { listener => + listener ! ResultListener.GridResult( + jointGrid, + msgAdapters.resultListener + ) } } } diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index c9dfe83b..7f7690e1 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -26,7 +26,10 @@ object ResultListener { with ResultEvent sealed trait Response - final case class ResultHandled(runId: UUID) extends Response + final case class ResultHandled( + runId: UUID, + replyTo: ActorRef[ResultListener.ResultEvent] + ) extends Response /* internal API */ sealed trait ResultEvent @@ -36,7 +39,7 @@ object ResultListener { .receive[ResultEvent] { case (ctx, GridResult(grid, replyTo)) => ctx.log.info(s"Received grid result for run id '${runId.toString}'") // TODO: Actual persistence and stuff, closing sinks, ... - replyTo ! ResultHandled(runId) + replyTo ! ResultHandled(runId, ctx.self) Behaviors.same } .receiveSignal { case (ctx, PostStop) => diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala index dd671455..2c88f598 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala @@ -10,12 +10,6 @@ import akka.actor.testkit.typed.Effect.MessageAdapter import akka.actor.testkit.typed.scaladsl.{ActorTestKit, TestProbe} import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.osmogrid.cfg.{OsmoGridConfig, OsmoGridConfigFactory} -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ - GuardianData, - MessageAdapters, - RunData -} -import edu.ie3.osmogrid.guardian.SubGridHandling import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator @@ -87,7 +81,7 @@ class SubGridHandlingSpec val runId = UUID.randomUUID() val grids = Range(1, 10).map(mockSubGrid) - val messageAdapters = new OsmoGridGuardian.MessageAdapters( + val messageAdapters = new RunGuardian.MessageAdapters( lvCoordinatorAdapter.ref, resultListenerAdapter.ref ) @@ -100,20 +94,13 @@ class SubGridHandlingSpec }.get "having an active run" should { - val guardianData = GuardianData( - messageAdapters, - Map( - runId -> RunData.Running( - runId, - cfg, - Some(resultListener.ref), - Seq(additionalResultListener.ref), - inputDataProvider.ref - ) - ) - ) "inform the right parties about correct information" in new SubGridHandling { - handleLvResults(runId, grids, guardianData) + handleLvResults( + grids, + cfg.generation, + Seq(resultListener.ref, additionalResultListener.ref), + messageAdapters + ) resultListener.receiveMessage() match { case ResultListener.GridResult(grid, _) => @@ -130,39 +117,6 @@ class SubGridHandlingSpec resultListenerAdapter.expectNoMessage() } } - - "having a run in coordinated shutdown phase" should { - val guardianData = GuardianData( - messageAdapters, - Map( - runId -> RunData.Stopping( - runId, - cfg - ) - ) - ) - "inform nobody about anything" in new SubGridHandling { - resultListener.expectNoMessage() - additionalResultListener.expectNoMessage() - inputDataProvider.expectNoMessage() - lvCoordinatorAdapter.expectNoMessage() - resultListenerAdapter.expectNoMessage() - } - } - - "having no matching run" should { - val guardianData = GuardianData( - messageAdapters, - Map.empty[UUID, RunData] - ) - "inform nobody about anything" in new SubGridHandling { - resultListener.expectNoMessage() - additionalResultListener.expectNoMessage() - inputDataProvider.expectNoMessage() - lvCoordinatorAdapter.expectNoMessage() - resultListenerAdapter.expectNoMessage() - } - } } } From 7c7713f3a981bc68816b171820deb7b171c1b95c Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 21 Jan 2022 14:16:41 +0100 Subject: [PATCH 21/38] Address first reviewer's remarks --- CHANGELOG.md | 2 +- build.gradle | 1 - src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala | 2 +- .../scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala | 1 - src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala | 3 ++- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10b055f6..d782a418 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Generation - Let `OsmoGridGuardian` handle multiple runs and spawn children accordingly - A `RunGuardian` takes care of a distinct simulation run and spawns all its needed services - - Spawn an `InputDataProvider` and a `ResultListener`(if need be) per run + - Spawn an `InputDataProvider` and a `ResultListener`(if required) per run - Spawn `LvCoordinator` and trigger it - Coordinated shut down phase - Only terminate OSMoGrid internal result event listener and let additional listeners alive diff --git a/build.gradle b/build.gradle index 64b58416..7770c879 100644 --- a/build.gradle +++ b/build.gradle @@ -105,7 +105,6 @@ dependencies { implementation 'math.geom2d:javaGeom:0.11.1' implementation 'info.picocli:picocli:4.6.2' // command line interface - spotbugsPlugins 'com.h3xstream.findsecbugs:findsecbugs-plugin:1.11.0' spotbugsPlugins 'com.h3xstream.findsecbugs:findsecbugs-plugin:1.11.0' // testing diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala index 949667eb..46150bf0 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala @@ -110,7 +110,7 @@ object ConfigFailFast extends LazyLogging { ) case Output(None) => throw IllegalConfigException( - "You have to provide at least one output data type!" + "You have to provide at least one output data sink, e.g. to .csv-files!" ) } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index c5c0fa04..96f410ee 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -29,7 +29,6 @@ import scala.util.{Failure, Success, Try} object OsmoGridGuardian { - /* Messages, that are understood and sent */ sealed trait Request /** Message to initiate a grid generation run diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala index a1552172..957db80e 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala @@ -10,6 +10,7 @@ import akka.actor.typed.ActorRef import akka.actor.typed.scaladsl.ActorContext import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} +import edu.ie3.osmogrid.exception.UnsupportedRequestException import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator @@ -71,7 +72,7 @@ trait RunSupport { s"Received unsupported grid generation config '$unsupported'. Stopping run with id '$runId'!" ) Failure( - UnsupportedOperationException( + UnsupportedRequestException( s"Unable to issue a generation run with the given parameters: '${cfg.generation}'" ) ) From 0ef9449eeb01609d7c8dde4d6d3ee4689b964df8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 21 Jan 2022 14:20:31 +0100 Subject: [PATCH 22/38] Move RunGuardian to distinct package --- .../osmogrid/guardian/OsmoGridGuardian.scala | 1 + .../guardian/{ => run}/RunGuardian.scala | 32 ++++++++++--------- .../guardian/{ => run}/RunSupport.scala | 5 +-- .../guardian/{ => run}/StopSupport.scala | 7 ++-- .../guardian/{ => run}/SubGridHandling.scala | 5 +-- .../{ => run}/SubGridHandlingSpec.scala | 3 +- 6 files changed, 29 insertions(+), 24 deletions(-) rename src/main/scala/edu/ie3/osmogrid/guardian/{ => run}/RunGuardian.scala (91%) rename src/main/scala/edu/ie3/osmogrid/guardian/{ => run}/RunSupport.scala (98%) rename src/main/scala/edu/ie3/osmogrid/guardian/{ => run}/StopSupport.scala (94%) rename src/main/scala/edu/ie3/osmogrid/guardian/{ => run}/SubGridHandling.scala (92%) rename src/test/scala/edu/ie3/osmogrid/guardian/{ => run}/SubGridHandlingSpec.scala (97%) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index 96f410ee..a198be56 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -16,6 +16,7 @@ import edu.ie3.datamodel.models.input.container.{ import edu.ie3.datamodel.utils.ContainerUtils import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} +import edu.ie3.osmogrid.guardian.run.RunGuardian import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, Request} diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala similarity index 91% rename from src/main/scala/edu/ie3/osmogrid/guardian/RunGuardian.scala rename to src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala index bfc3a0a1..f6de1c25 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/RunGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala @@ -4,16 +4,13 @@ * Research group Distribution grid planning and operation */ -package edu.ie3.osmogrid.guardian +package edu.ie3.osmogrid.guardian.run +import akka.actor.typed.scaladsl.Behaviors import akka.actor.typed.{ActorRef, Behavior} -import akka.actor.typed.scaladsl.{Behaviors, StashBuffer} import edu.ie3.osmogrid.cfg.OsmoGridConfig -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.Request -import edu.ie3.osmogrid.guardian.RunGuardian.MessageAdapters.{ - WrappedListenerResponse, - WrappedLvCoordinatorResponse -} +import edu.ie3.osmogrid.guardian.run.RunGuardian.MessageAdapters.{WrappedListenerResponse, WrappedLvCoordinatorResponse} +import edu.ie3.osmogrid.guardian.run.{RunSupport, StopSupport, SubGridHandling} import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator @@ -25,6 +22,7 @@ import scala.util.{Failure, Success} */ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { sealed trait Request + object Run extends Request /** Container object with all available adapters for outside protocol messages @@ -34,11 +32,12 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { * @param resultListener * Adapter for messages from [[ResultEventListener]] */ - private[guardian] final case class MessageAdapters( + private[run] final case class MessageAdapters( lvCoordinator: ActorRef[LvCoordinator.Response], resultListener: ActorRef[ResultListener.Response] ) - private[guardian] object MessageAdapters { + + private[run] object MessageAdapters { final case class WrappedLvCoordinatorResponse( response: LvCoordinator.Response ) extends Request @@ -49,14 +48,16 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { } sealed trait Response + final case class Done(runId: UUID) extends Response sealed trait Watch extends Request - private[guardian] object InputDataProviderDied extends Watch - private[guardian] object ResultEventListenerDied extends Watch + private[run] object InputDataProviderDied extends Watch + + private[run] object ResultEventListenerDied extends Watch - private[guardian] object LvCoordinatorDied extends Watch + private[run] object LvCoordinatorDied extends Watch final case class RunGuardianData( runId: UUID, @@ -65,7 +66,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { msgAdapters: MessageAdapters ) - private[guardian] final case class ChildReferences( + private[run] final case class ChildReferences( inputDataProvider: ActorRef[InputDataProvider.Request], resultListener: Option[ActorRef[ResultListener.ResultEvent]], additionalResultListeners: Seq[ActorRef[ResultListener.ResultEvent]], @@ -87,7 +88,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { * @param lvCoordinatorTerminated * Optional information, if the [[LvCoordinator]] has stopped */ - private[guardian] final case class StoppingData( + private[run] final case class StoppingData( inputDataProviderTerminated: Boolean, resultListenerTerminated: Boolean, lvCoordinatorTerminated: Option[Boolean] @@ -135,7 +136,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { * @return * the next state */ - private[guardian] def idle( + private[run] def idle( runId: UUID, cfg: OsmoGridConfig, additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], @@ -159,6 +160,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { } /** Behavior to indicate, that a simulation run is currently active + * * @param runId * Identifier of the run * @param cfg diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala similarity index 98% rename from src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala rename to src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala index 957db80e..44e694be 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala @@ -4,13 +4,14 @@ * Research group Distribution grid planning and operation */ -package edu.ie3.osmogrid.guardian +package edu.ie3.osmogrid.guardian.run import akka.actor.typed.ActorRef import akka.actor.typed.scaladsl.ActorContext -import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} +import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.exception.UnsupportedRequestException +import edu.ie3.osmogrid.guardian.run.RunGuardian import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala similarity index 94% rename from src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala rename to src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala index 4140e980..e5a6714d 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/StopSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala @@ -4,12 +4,11 @@ * Research group Distribution grid planning and operation */ -package edu.ie3.osmogrid.guardian +package edu.ie3.osmogrid.guardian.run -import akka.actor.typed.Behavior import akka.actor.typed.scaladsl.ActorContext -import edu.ie3.osmogrid.guardian.RunGuardian.{ChildReferences, StoppingData} -import org.slf4j.Logger +import edu.ie3.osmogrid.guardian.run.RunGuardian +import edu.ie3.osmogrid.guardian.run.RunGuardian.{ChildReferences, StoppingData} import java.util.UUID diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala similarity index 92% rename from src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala rename to src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala index 2c0bd41c..90a13fc5 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala @@ -4,13 +4,14 @@ * Research group Distribution grid planning and operation */ -package edu.ie3.osmogrid.guardian +package edu.ie3.osmogrid.guardian.run import akka.actor.typed.ActorRef import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.datamodel.utils.ContainerUtils import edu.ie3.osmogrid.cfg.OsmoGridConfig -import edu.ie3.osmogrid.guardian.SubGridHandling.assignSubnetNumbers +import edu.ie3.osmogrid.guardian.run.RunGuardian +import edu.ie3.osmogrid.guardian.run.SubGridHandling.assignSubnetNumbers import edu.ie3.osmogrid.io.output.ResultListener import org.slf4j.Logger diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/run/SubGridHandlingSpec.scala similarity index 97% rename from src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala rename to src/test/scala/edu/ie3/osmogrid/guardian/run/SubGridHandlingSpec.scala index 2c88f598..d2196912 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/SubGridHandlingSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/run/SubGridHandlingSpec.scala @@ -4,12 +4,13 @@ * Research group Distribution grid planning and operation */ -package edu.ie3.osmogrid.guardian +package edu.ie3.osmogrid.guardian.run import akka.actor.testkit.typed.Effect.MessageAdapter import akka.actor.testkit.typed.scaladsl.{ActorTestKit, TestProbe} import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.osmogrid.cfg.{OsmoGridConfig, OsmoGridConfigFactory} +import edu.ie3.osmogrid.guardian.run.{RunGuardian, SubGridHandling} import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator From bdb0366aa38fc879540aabbd79c873e714585b4f Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 21 Jan 2022 14:34:38 +0100 Subject: [PATCH 23/38] Introduce package object for shared classes among the package --- .../osmogrid/guardian/OsmoGridGuardian.scala | 3 +- .../osmogrid/guardian/run/RunGuardian.scala | 77 ++--------------- .../osmogrid/guardian/run/RunSupport.scala | 22 ++--- .../osmogrid/guardian/run/StopSupport.scala | 23 +++-- .../guardian/run/SubGridHandling.scala | 2 +- .../ie3/osmogrid/guardian/run/package.scala | 86 +++++++++++++++++++ .../guardian/run/SubGridHandlingSpec.scala | 2 +- 7 files changed, 118 insertions(+), 97 deletions(-) create mode 100644 src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala index a198be56..4a1011b8 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardian.scala @@ -16,6 +16,7 @@ import edu.ie3.datamodel.models.input.container.{ import edu.ie3.datamodel.utils.ContainerUtils import edu.ie3.osmogrid.cfg.{ConfigFailFast, OsmoGridConfig} import edu.ie3.osmogrid.cfg.OsmoGridConfig.{Generation, Output} +import edu.ie3.osmogrid.guardian.run.Run import edu.ie3.osmogrid.guardian.run.RunGuardian import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener @@ -82,7 +83,7 @@ object OsmoGridGuardian { s"RunGuardian_$runId" ) ctx.watchWith(runGuardian, RunGuardianDied(runId)) - runGuardian ! RunGuardian.Run + runGuardian ! run.Run idle(guardianData.append(runId)) case (ctx, watch: Watch) => diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala index f6de1c25..e8fdc99f 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala @@ -9,7 +9,11 @@ package edu.ie3.osmogrid.guardian.run import akka.actor.typed.scaladsl.Behaviors import akka.actor.typed.{ActorRef, Behavior} import edu.ie3.osmogrid.cfg.OsmoGridConfig -import edu.ie3.osmogrid.guardian.run.RunGuardian.MessageAdapters.{WrappedListenerResponse, WrappedLvCoordinatorResponse} +import edu.ie3.osmogrid.guardian.run.MessageAdapters +import edu.ie3.osmogrid.guardian.run.MessageAdapters.{ + WrappedListenerResponse, + WrappedLvCoordinatorResponse +} import edu.ie3.osmogrid.guardian.run.{RunSupport, StopSupport, SubGridHandling} import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener @@ -21,83 +25,14 @@ import scala.util.{Failure, Success} /** Actor to take care of a specific simulation run */ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { - sealed trait Request - - object Run extends Request - - /** Container object with all available adapters for outside protocol messages - * - * @param lvCoordinator - * Adapter for messages from [[LvCoordinator]] - * @param resultListener - * Adapter for messages from [[ResultEventListener]] - */ - private[run] final case class MessageAdapters( - lvCoordinator: ActorRef[LvCoordinator.Response], - resultListener: ActorRef[ResultListener.Response] - ) - - private[run] object MessageAdapters { - final case class WrappedLvCoordinatorResponse( - response: LvCoordinator.Response - ) extends Request - - final case class WrappedListenerResponse( - response: ResultListener.Response - ) extends Request - } - - sealed trait Response - - final case class Done(runId: UUID) extends Response - sealed trait Watch extends Request - - private[run] object InputDataProviderDied extends Watch - - private[run] object ResultEventListenerDied extends Watch - - private[run] object LvCoordinatorDied extends Watch - - final case class RunGuardianData( + private final case class RunGuardianData( runId: UUID, cfg: OsmoGridConfig, additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], msgAdapters: MessageAdapters ) - private[run] final case class ChildReferences( - inputDataProvider: ActorRef[InputDataProvider.Request], - resultListener: Option[ActorRef[ResultListener.ResultEvent]], - additionalResultListeners: Seq[ActorRef[ResultListener.ResultEvent]], - lvCoordinator: Option[ActorRef[LvCoordinator.Request]] - ) { - def resultListeners: Seq[ActorRef[ResultListener.ResultEvent]] = - resultListener - .map(Seq(_)) - .getOrElse(Seq.empty) ++ additionalResultListeners - } - - /** Meta data to keep track of which children already terminated during the - * coordinated shutdown phase - * - * @param inputDataProviderTerminated - * If the [[InputDataProvider]] has stopped - * @param resultListenerTerminated - * If the [[ResultListener]] has stopped - * @param lvCoordinatorTerminated - * Optional information, if the [[LvCoordinator]] has stopped - */ - private[run] final case class StoppingData( - inputDataProviderTerminated: Boolean, - resultListenerTerminated: Boolean, - lvCoordinatorTerminated: Option[Boolean] - ) { - def allChildrenTerminated: Boolean = - inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated - .contains(true) - } - /** Instantiate the actor * * @param cfg diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala index 44e694be..8c95d0c5 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala @@ -41,9 +41,9 @@ trait RunSupport { runId: UUID, cfg: OsmoGridConfig, additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], - msgAdapters: RunGuardian.MessageAdapters, - ctx: ActorContext[RunGuardian.Request] - ): Try[RunGuardian.ChildReferences] = { + msgAdapters: MessageAdapters, + ctx: ActorContext[Request] + ): Try[ChildReferences] = { val log = ctx.log ConfigFailFast.check(cfg, additionalListener) log.info(s"Initializing grid generation for run with id '$runId'!") @@ -61,7 +61,7 @@ trait RunSupport { ctx ) Success( - RunGuardian.ChildReferences( + ChildReferences( inputProvider, resultEventListener, additionalListener, @@ -97,7 +97,7 @@ trait RunSupport { runId: UUID, inputConfig: OsmoGridConfig.Input, outputConfig: OsmoGridConfig.Output, - ctx: ActorContext[RunGuardian.Request] + ctx: ActorContext[Request] ): ( ActorRef[InputDataProvider.Request], Option[ActorRef[ResultListener.ResultEvent]] @@ -120,7 +120,7 @@ trait RunSupport { private def spawnInputDataProvider( runId: UUID, inputConfig: OsmoGridConfig.Input, - ctx: ActorContext[RunGuardian.Request] + ctx: ActorContext[Request] ): ActorRef[InputDataProvider.Request] = { ctx.log.info("Starting input data provider ...") val inputProvider = @@ -128,7 +128,7 @@ trait RunSupport { InputDataProvider(inputConfig), s"InputDataProvider_${runId.toString}" ) - ctx.watchWith(inputProvider, RunGuardian.InputDataProviderDied) + ctx.watchWith(inputProvider, InputDataProviderDied) inputProvider } @@ -146,7 +146,7 @@ trait RunSupport { private def spawnResultListener( runId: UUID, outputConfig: OsmoGridConfig.Output, - ctx: ActorContext[RunGuardian.Request] + ctx: ActorContext[Request] ): Option[ActorRef[ResultListener.ResultEvent]] = { val resultListener = outputConfig match { case Output(Some(_)) => @@ -162,7 +162,7 @@ trait RunSupport { None } resultListener.foreach( - ctx.watchWith(_, RunGuardian.ResultEventListenerDied) + ctx.watchWith(_, ResultEventListenerDied) ) resultListener } @@ -183,11 +183,11 @@ trait RunSupport { runId: UUID, lvConfig: OsmoGridConfig.Generation.Lv, lvCoordinatorAdapter: ActorRef[LvCoordinator.Response], - ctx: ActorContext[RunGuardian.Request] + ctx: ActorContext[Request] ): ActorRef[LvCoordinator.Request] = { val lvCoordinator = ctx.spawn(LvCoordinator(), s"LvCoordinator_${runId.toString}") - ctx.watchWith(lvCoordinator, RunGuardian.LvCoordinatorDied) + ctx.watchWith(lvCoordinator, LvCoordinatorDied) ctx.log.info("Starting voltage level grid generation ...") lvCoordinator ! ReqLvGrids(runId, lvConfig, lvCoordinatorAdapter) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala index e5a6714d..b7881639 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala @@ -8,7 +8,6 @@ package edu.ie3.osmogrid.guardian.run import akka.actor.typed.scaladsl.ActorContext import edu.ie3.osmogrid.guardian.run.RunGuardian -import edu.ie3.osmogrid.guardian.run.RunGuardian.{ChildReferences, StoppingData} import java.util.UUID @@ -23,8 +22,8 @@ trait StopSupport { * Current actor context */ protected def stopChildren( - childReferences: RunGuardian.ChildReferences, - ctx: ActorContext[RunGuardian.Request] + childReferences: ChildReferences, + ctx: ActorContext[Request] ): StoppingData = { ctx.stop(childReferences.inputDataProvider) childReferences.resultListener.foreach(ctx.stop) @@ -42,14 +41,14 @@ trait StopSupport { * Next state with updated [[GuardianData]] */ protected def registerCoordinatedShutDown( - watchMsg: RunGuardian.Watch, + watchMsg: Watch, stoppingData: StoppingData ): StoppingData = watchMsg match { - case RunGuardian.InputDataProviderDied => + case InputDataProviderDied => stoppingData.copy(inputDataProviderTerminated = true) - case RunGuardian.ResultEventListenerDied => + case ResultEventListenerDied => stoppingData.copy(resultListenerTerminated = true) - case RunGuardian.LvCoordinatorDied => + case LvCoordinatorDied => stoppingData.copy(lvCoordinatorTerminated = stoppingData.lvCoordinatorTerminated.map(_ => true) ) @@ -72,21 +71,21 @@ trait StopSupport { protected def handleUnexpectedShutDown( runId: UUID, childReferences: ChildReferences, - watchMsg: RunGuardian.Watch, - ctx: ActorContext[RunGuardian.Request] + watchMsg: Watch, + ctx: ActorContext[Request] ): StoppingData = { (stopChildren(childReferences, ctx), watchMsg) match { - case (stoppingData, RunGuardian.InputDataProviderDied) => + case (stoppingData, InputDataProviderDied) => ctx.log.warn( s"Input data provider for run $runId unexpectedly died. Start coordinated shut down phase for this run." ) stoppingData.copy(inputDataProviderTerminated = true) - case (stoppingData, RunGuardian.ResultEventListenerDied) => + case (stoppingData, ResultEventListenerDied) => ctx.log.warn( s"One of the result listener for run $runId unexpectedly died. Start coordinated shut down phase for this run." ) stoppingData.copy(resultListenerTerminated = true) - case (stoppingData, RunGuardian.LvCoordinatorDied) => + case (stoppingData, LvCoordinatorDied) => ctx.log.warn( s"Lv coordinator for run $runId unexpectedly died. Start coordinated shut down phase for this run." ) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala index 90a13fc5..1bba81d2 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala @@ -35,7 +35,7 @@ trait SubGridHandling { grids: Seq[SubGridContainer], cfg: OsmoGridConfig.Generation, resultListener: Seq[ActorRef[ResultListener.ResultEvent]], - msgAdapters: RunGuardian.MessageAdapters + msgAdapters: MessageAdapters )(implicit log: Logger): Unit = { log.info("All lv grids successfully generated.") val updatedSubGrids = assignSubnetNumbers(grids) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala new file mode 100644 index 00000000..6274700c --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala @@ -0,0 +1,86 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian + +import akka.actor.typed.ActorRef +import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener +import edu.ie3.osmogrid.lv.LvCoordinator + +import java.util.UUID + +package object run { + sealed trait Request + + object Run extends Request + + /** Container object with all available adapters for outside protocol messages + * + * @param lvCoordinator + * Adapter for messages from [[LvCoordinator]] + * @param resultListener + * Adapter for messages from [[ResultEventListener]] + */ + private final case class MessageAdapters( + lvCoordinator: ActorRef[LvCoordinator.Response], + resultListener: ActorRef[ResultListener.Response] + ) + + private object MessageAdapters { + final case class WrappedLvCoordinatorResponse( + response: LvCoordinator.Response + ) extends Request + + final case class WrappedListenerResponse( + response: ResultListener.Response + ) extends Request + } + + sealed trait Response + + final case class Done(runId: UUID) extends Response + + sealed trait Watch extends Request + + private object InputDataProviderDied extends Watch + + private object ResultEventListenerDied extends Watch + + private object LvCoordinatorDied extends Watch + + private final case class ChildReferences( + inputDataProvider: ActorRef[InputDataProvider.Request], + resultListener: Option[ActorRef[ResultListener.ResultEvent]], + additionalResultListeners: Seq[ActorRef[ResultListener.ResultEvent]], + lvCoordinator: Option[ActorRef[LvCoordinator.Request]] + ) { + def resultListeners: Seq[ActorRef[ResultListener.ResultEvent]] = + resultListener + .map(Seq(_)) + .getOrElse(Seq.empty) ++ additionalResultListeners + } + + /** Meta data to keep track of which children already terminated during the + * coordinated shutdown phase + * + * @param inputDataProviderTerminated + * If the [[InputDataProvider]] has stopped + * @param resultListenerTerminated + * If the [[ResultListener]] has stopped + * @param lvCoordinatorTerminated + * Optional information, if the [[LvCoordinator]] has stopped + */ + private final case class StoppingData( + inputDataProviderTerminated: Boolean, + resultListenerTerminated: Boolean, + lvCoordinatorTerminated: Option[Boolean] + ) { + def allChildrenTerminated: Boolean = + inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated + .contains(true) + } +} diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/run/SubGridHandlingSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/run/SubGridHandlingSpec.scala index d2196912..e6e8b3d7 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/run/SubGridHandlingSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/run/SubGridHandlingSpec.scala @@ -82,7 +82,7 @@ class SubGridHandlingSpec val runId = UUID.randomUUID() val grids = Range(1, 10).map(mockSubGrid) - val messageAdapters = new RunGuardian.MessageAdapters( + val messageAdapters = new MessageAdapters( lvCoordinatorAdapter.ref, resultListenerAdapter.ref ) From 11c40a1398b4e538023ca0765aec193b8be4b1db Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 21 Jan 2022 14:38:32 +0100 Subject: [PATCH 24/38] No need to forward run uuid --- .../scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala | 2 +- .../scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala | 2 +- src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala index e8fdc99f..15dd306a 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala @@ -116,7 +116,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { case ( ctx, WrappedLvCoordinatorResponse( - LvCoordinator.RepLvGrids(_, subGridContainers) + LvCoordinator.RepLvGrids(subGridContainers) ) ) => /* Handle the grid results and wait for the listener to report back */ diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala index 8c95d0c5..7dd75151 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala @@ -190,7 +190,7 @@ trait RunSupport { ctx.watchWith(lvCoordinator, LvCoordinatorDied) ctx.log.info("Starting voltage level grid generation ...") - lvCoordinator ! ReqLvGrids(runId, lvConfig, lvCoordinatorAdapter) + lvCoordinator ! ReqLvGrids(lvConfig, lvCoordinatorAdapter) lvCoordinator } diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala index a63fd137..61245a85 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala @@ -18,21 +18,18 @@ import java.util.UUID object LvCoordinator { sealed trait Request final case class ReqLvGrids( - runId: UUID, cfg: OsmoGridConfig.Generation.Lv, replyTo: ActorRef[Response] ) extends Request sealed trait Response - final case class RepLvGrids(runId: UUID, grids: Seq[SubGridContainer]) - extends Response + final case class RepLvGrids(grids: Seq[SubGridContainer]) extends Response def apply(): Behavior[Request] = idle private def idle: Behavior[Request] = Behaviors.receive { (ctx, msg) => msg match { case ReqLvGrids( - runId, Lv( amountOfGridGenerators, amountOfRegionCoordinators, @@ -68,7 +65,7 @@ object LvCoordinator { val lvRegionCoordinatorProxy = ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") - replyTo ! RepLvGrids(runId, Seq.empty[SubGridContainer]) + replyTo ! RepLvGrids(Seq.empty[SubGridContainer]) Behaviors.stopped case unsupported => ctx.log.error(s"Received unsupported message: $unsupported") From e96dacddc64ec670cd3cb9b285a4756b352d27ed Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Fri, 21 Jan 2022 15:12:41 +0100 Subject: [PATCH 25/38] Passing around state data instead of single parameters --- .../osmogrid/guardian/run/RunGuardian.scala | 98 ++++++++----------- .../osmogrid/guardian/run/RunSupport.scala | 43 ++++---- .../osmogrid/guardian/run/StopSupport.scala | 12 ++- .../ie3/osmogrid/guardian/run/package.scala | 14 ++- 4 files changed, 88 insertions(+), 79 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala index 15dd306a..7e21085d 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala @@ -26,13 +26,6 @@ import scala.util.{Failure, Success} */ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { - private final case class RunGuardianData( - runId: UUID, - cfg: OsmoGridConfig, - additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], - msgAdapters: MessageAdapters - ) - /** Instantiate the actor * * @param cfg @@ -48,70 +41,64 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { runId: UUID ): Behavior[Request] = Behaviors.setup { ctx => idle( - runId, - cfg, - additionalListener, - MessageAdapters( - ctx.messageAdapter(msg => WrappedLvCoordinatorResponse(msg)), - ctx.messageAdapter(msg => WrappedListenerResponse(msg)) + RunGuardianData( + runId, + cfg, + additionalListener, + MessageAdapters( + ctx.messageAdapter(msg => WrappedLvCoordinatorResponse(msg)), + ctx.messageAdapter(msg => WrappedListenerResponse(msg)) + ) ) ) } /** This actor is in idle state and waits for any kind of request * - * @param runId - * Identifier of the current run - * @param cfg - * Configuration for this run - * @param additionalListener - * References to additional [[ResultListener]]s - * @param msgAdapters - * Collection of all message adapters + * @param runGuardianData + * Meta information describing the current actor's state * @return * the next state */ - private[run] def idle( - runId: UUID, - cfg: OsmoGridConfig, - additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], - msgAdapters: MessageAdapters - ): Behavior[Request] = + private def idle(runGuardianData: RunGuardianData): Behavior[Request] = Behaviors.receive { case (ctx, Run) => /* Start a run */ - initRun(runId, cfg, additionalListener, msgAdapters, ctx) match { + initRun( + runGuardianData, + ctx + ) match { case Success(childReferences) => - running(runId, cfg, childReferences, msgAdapters) + running( + runGuardianData, + childReferences + ) case Failure(exception) => - ctx.log.error(s"Unable to start run $runId.", exception) + ctx.log.error( + s"Unable to start run ${runGuardianData.runId}.", + exception + ) Behaviors.stopped } case (ctx, notUnderstood) => ctx.log.error( - s"Received a message, that I don't understand during idle phase of run $runId.\n\tMessage: $notUnderstood" + s"Received a message, that I don't understand during idle phase of run ${runGuardianData.runId}.\n\tMessage: $notUnderstood" ) Behaviors.same } /** Behavior to indicate, that a simulation run is currently active * - * @param runId - * Identifier of the run - * @param cfg - * Configuration for this run + * @param runGuardianData + * Meta information describing the current actor's state * @param childReferences * References to child actors - * @param msgAdapters - * Collection of message adapters * @return * The next state */ private def running( - runId: UUID, - cfg: OsmoGridConfig, - childReferences: ChildReferences, - msgAdapters: MessageAdapters + runGuardianData: RunGuardianData, + childReferences: ChildReferences ): Behavior[Request] = Behaviors.receive { case ( ctx, @@ -122,9 +109,9 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { /* Handle the grid results and wait for the listener to report back */ handleLvResults( subGridContainers, - cfg.generation, + runGuardianData.cfg.generation, childReferences.resultListeners, - msgAdapters + runGuardianData.msgAdapters )(ctx.log) Behaviors.same case ( @@ -132,7 +119,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { WrappedListenerResponse(ResultListener.ResultHandled(_, sender)) ) => ctx.log.debug( - s"The listener $sender has successfully handled the result event of run $runId." + s"The listener $sender has successfully handled the result event of run ${runGuardianData.runId}." ) if ( childReferences.resultListener.contains( @@ -141,9 +128,9 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { ) { /* Start coordinated shutdown */ ctx.log.info( - s"Run $runId successfully finished. Stop all run-related processes." + s"Run ${runGuardianData.runId} successfully finished. Stop all run-related processes." ) - stopping(runId, stopChildren(childReferences, ctx)) + stopping(stopChildren(runGuardianData.runId, childReferences, ctx)) } else { /* Somebody did something great, but nothing, that affects us */ Behaviors.same @@ -151,12 +138,16 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { case (ctx, watch: Watch) => /* Somebody died unexpectedly. Start coordinated shutdown */ stopping( - runId, - handleUnexpectedShutDown(runId, childReferences, watch, ctx) + handleUnexpectedShutDown( + runGuardianData.runId, + childReferences, + watch, + ctx + ) ) case (ctx, notUnderstood) => ctx.log.error( - s"Received a message, that I don't understand during active run $runId.\n\tMessage: $notUnderstood" + s"Received a message, that I don't understand during active run ${runGuardianData.runId}.\n\tMessage: $notUnderstood" ) Behaviors.same } @@ -164,29 +155,26 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { /** Behavior, that indicates, that a coordinate shutdown of the children takes * place * - * @param runId - * Identifier of the run * @param stoppingData * Information about who already has terminated * @return * The next state */ private def stopping( - runId: UUID, stoppingData: StoppingData ): Behavior[Request] = Behaviors.receive { case (ctx, watch: Watch) => val updatedStoppingData = registerCoordinatedShutDown(watch, stoppingData) if (updatedStoppingData.allChildrenTerminated) { ctx.log.info( - s"All child processes of run $runId successfully terminated. Finally terminating the whole run process." + s"All child processes of run ${stoppingData.runId} successfully terminated. Finally terminating the whole run process." ) /* The overall guardian is automatically informed via death watch */ Behaviors.stopped - } else stopping(runId, updatedStoppingData) + } else stopping(updatedStoppingData) case (ctx, notUnderstood) => ctx.log.error( - s"Received a message, that I don't understand during coordinated shutdown phase of run $runId.\n\tMessage: $notUnderstood" + s"Received a message, that I don't understand during coordinated shutdown phase of run ${stoppingData.runId}.\n\tMessage: $notUnderstood" ) Behaviors.same } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala index 7dd75151..63f2f5e4 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala @@ -24,57 +24,58 @@ trait RunSupport { /** Initiate a generation run and return the updated run meta data * - * @param runId - * Identifier of the run - * @param cfg - * Configuration for the run - * @param additionalListener - * References to additional listeners - * @param msgAdapters - * Collection of all message adapters + * @param runGuardianData + * Meta information describing the current actor's state * @param ctx * Current actor context * @return * Updated run meta data */ protected def initRun( - runId: UUID, - cfg: OsmoGridConfig, - additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], - msgAdapters: MessageAdapters, + runGuardianData: RunGuardianData, ctx: ActorContext[Request] ): Try[ChildReferences] = { val log = ctx.log - ConfigFailFast.check(cfg, additionalListener) - log.info(s"Initializing grid generation for run with id '$runId'!") + ConfigFailFast.check( + runGuardianData.cfg, + runGuardianData.additionalListener + ) + log.info( + s"Initializing grid generation for run with id '${runGuardianData.runId}'!" + ) /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ - cfg.generation match { + runGuardianData.cfg.generation match { case Generation(Some(lvConfig)) => ctx.log.info("Starting low voltage grid coordinator ...") val (inputProvider, resultEventListener) = - spawnIoActors(runId, cfg.input, cfg.output, ctx) + spawnIoActors( + runGuardianData.runId, + runGuardianData.cfg.input, + runGuardianData.cfg.output, + ctx + ) val lvCoordinator = startLvGridGeneration( - runId, + runGuardianData.runId, lvConfig, - msgAdapters.lvCoordinator, + runGuardianData.msgAdapters.lvCoordinator, ctx ) Success( ChildReferences( inputProvider, resultEventListener, - additionalListener, + runGuardianData.additionalListener, Some(lvCoordinator) ) ) case unsupported => ctx.log.error( - s"Received unsupported grid generation config '$unsupported'. Stopping run with id '$runId'!" + s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${runGuardianData.runId}'!" ) Failure( UnsupportedRequestException( - s"Unable to issue a generation run with the given parameters: '${cfg.generation}'" + s"Unable to issue a generation run with the given parameters: '${runGuardianData.cfg.generation}'" ) ) } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala index b7881639..ff41ffc3 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala @@ -16,19 +16,27 @@ trait StopSupport { /** Stop all children for the given run. The additional listeners are not * asked to be stopped! * + * @param runId + * Identifier of the current run * @param childReferences * References to children * @param ctx * Current actor context */ protected def stopChildren( + runId: UUID, childReferences: ChildReferences, ctx: ActorContext[Request] ): StoppingData = { ctx.stop(childReferences.inputDataProvider) childReferences.resultListener.foreach(ctx.stop) - StoppingData(false, false, childReferences.lvCoordinator.map(_ => false)) + StoppingData( + runId, + false, + false, + childReferences.lvCoordinator.map(_ => false) + ) } /** Register [[Watch]] messages within the coordinated shutdown phase of a run @@ -74,7 +82,7 @@ trait StopSupport { watchMsg: Watch, ctx: ActorContext[Request] ): StoppingData = { - (stopChildren(childReferences, ctx), watchMsg) match { + (stopChildren(runId, childReferences, ctx), watchMsg) match { case (stoppingData, InputDataProviderDied) => ctx.log.warn( s"Input data provider for run $runId unexpectedly died. Start coordinated shut down phase for this run." diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala index 6274700c..981c878f 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala @@ -7,6 +7,7 @@ package edu.ie3.osmogrid.guardian import akka.actor.typed.ActorRef +import edu.ie3.osmogrid.cfg.OsmoGridConfig import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.lv.LvCoordinator @@ -64,9 +65,19 @@ package object run { .getOrElse(Seq.empty) ++ additionalResultListeners } + private sealed trait StateData + private final case class RunGuardianData( + runId: UUID, + cfg: OsmoGridConfig, + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], + msgAdapters: MessageAdapters + ) extends StateData + /** Meta data to keep track of which children already terminated during the * coordinated shutdown phase * + * @param runId + * Identifier of the run * @param inputDataProviderTerminated * If the [[InputDataProvider]] has stopped * @param resultListenerTerminated @@ -75,10 +86,11 @@ package object run { * Optional information, if the [[LvCoordinator]] has stopped */ private final case class StoppingData( + runId: UUID, inputDataProviderTerminated: Boolean, resultListenerTerminated: Boolean, lvCoordinatorTerminated: Option[Boolean] - ) { + ) extends StateData { def allChildrenTerminated: Boolean = inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated .contains(true) From 4f4dfdf0a9c155a9fafbde89a1c010df869f520c Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 24 Jan 2022 09:59:41 +0100 Subject: [PATCH 26/38] Test spawning of RunGuardian --- .../osmogrid/cfg/OsmoGridConfigFactory.scala | 5 +++ .../guardian/OsmoGridGuardianSpec.scala | 44 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala diff --git a/src/test/scala/edu/ie3/osmogrid/cfg/OsmoGridConfigFactory.scala b/src/test/scala/edu/ie3/osmogrid/cfg/OsmoGridConfigFactory.scala index 78616642..6f1d437d 100644 --- a/src/test/scala/edu/ie3/osmogrid/cfg/OsmoGridConfigFactory.scala +++ b/src/test/scala/edu/ie3/osmogrid/cfg/OsmoGridConfigFactory.scala @@ -12,6 +12,11 @@ import java.io.File import scala.util.Try object OsmoGridConfigFactory { + lazy val defaultTestConfig: OsmoGridConfig = + OsmoGridConfig( + ConfigFactory.parseFile(new File("src/test/resources/testConfig.conf")) + ) + def parse(config: String): Try[OsmoGridConfig] = Try { OsmoGridConfig( ConfigFactory diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala new file mode 100644 index 00000000..ae874e0e --- /dev/null +++ b/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala @@ -0,0 +1,44 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian + +import akka.actor.testkit.typed.Effect.{MessageAdapter, Spawned} +import akka.actor.testkit.typed.scaladsl.BehaviorTestKit +import akka.actor.typed.{ActorRef, Behavior} +import edu.ie3.osmogrid.cfg.OsmoGridConfigFactory +import edu.ie3.osmogrid.guardian.run.RunGuardian +import edu.ie3.osmogrid.guardian.run.Request +import edu.ie3.osmogrid.io.output.ResultListener.ResultEvent +import edu.ie3.test.common.UnitSpec + +import java.util.UUID + +class OsmoGridGuardianSpec extends UnitSpec { + "Having an overall OsmoGridGuardian" when { + "being idle" should { + "spawn a new RunGuardian on request" in { + val guardianData = OsmoGridGuardian.GuardianData(Seq.empty[UUID]) + val config = OsmoGridConfigFactory.defaultTestConfig + val additionalListeners = Seq.empty[ActorRef[ResultEvent]] + val runId = UUID.randomUUID() + + val idleTestKit = BehaviorTestKit(OsmoGridGuardian.idle(guardianData)) + idleTestKit.run( + OsmoGridGuardian.Run(config, additionalListeners, runId) + ) + + /* Check if the right child is spawned */ + idleTestKit.expectEffectPF { + case Spawned(childBehav: Behavior[Request], name, props) => + name shouldBe s"RunGuardian_$runId" + case Spawned(childBehav, _, _) => + fail(s"Spawned a child with wrong behavior '$childBehav'.") + } + } + } + } +} From f8900a8126ec11d484781f4e91cd6378da8f4eda Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 24 Jan 2022 10:35:23 +0100 Subject: [PATCH 27/38] Check logging results for dead runs --- .../guardian/OsmoGridGuardianSpec.scala | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala index ae874e0e..0036f3f8 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala @@ -6,27 +6,31 @@ package edu.ie3.osmogrid.guardian +import akka.actor.testkit.typed.CapturedLogEvent import akka.actor.testkit.typed.Effect.{MessageAdapter, Spawned} import akka.actor.testkit.typed.scaladsl.BehaviorTestKit import akka.actor.typed.{ActorRef, Behavior} import edu.ie3.osmogrid.cfg.OsmoGridConfigFactory +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunGuardianDied import edu.ie3.osmogrid.guardian.run.RunGuardian import edu.ie3.osmogrid.guardian.run.Request import edu.ie3.osmogrid.io.output.ResultListener.ResultEvent import edu.ie3.test.common.UnitSpec +import org.slf4j.event.Level import java.util.UUID class OsmoGridGuardianSpec extends UnitSpec { "Having an overall OsmoGridGuardian" when { "being idle" should { - "spawn a new RunGuardian on request" in { - val guardianData = OsmoGridGuardian.GuardianData(Seq.empty[UUID]) - val config = OsmoGridConfigFactory.defaultTestConfig - val additionalListeners = Seq.empty[ActorRef[ResultEvent]] - val runId = UUID.randomUUID() + val guardianData = OsmoGridGuardian.GuardianData(Seq.empty[UUID]) + val config = OsmoGridConfigFactory.defaultTestConfig + val additionalListeners = Seq.empty[ActorRef[ResultEvent]] + val runId = UUID.randomUUID() + + val idleTestKit = BehaviorTestKit(OsmoGridGuardian.idle(guardianData)) - val idleTestKit = BehaviorTestKit(OsmoGridGuardian.idle(guardianData)) + "spawn a new RunGuardian on request" in { idleTestKit.run( OsmoGridGuardian.Run(config, additionalListeners, runId) ) @@ -39,6 +43,15 @@ class OsmoGridGuardianSpec extends UnitSpec { fail(s"Spawned a child with wrong behavior '$childBehav'.") } } + + "report a dead RunGuardian" in { + idleTestKit.run(RunGuardianDied(runId)) + + idleTestKit.logEntries() should contain only CapturedLogEvent( + Level.INFO, + s"Run $runId terminated." + ) + } } } } From ec72ca54f0bf56ce4983600fd45d47115551b3b2 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 24 Jan 2022 12:57:29 +0100 Subject: [PATCH 28/38] Testing the run data --- .../guardian/OsmoGridGuardianSpec.scala | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala index 0036f3f8..fc4b57aa 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/OsmoGridGuardianSpec.scala @@ -11,7 +11,10 @@ import akka.actor.testkit.typed.Effect.{MessageAdapter, Spawned} import akka.actor.testkit.typed.scaladsl.BehaviorTestKit import akka.actor.typed.{ActorRef, Behavior} import edu.ie3.osmogrid.cfg.OsmoGridConfigFactory -import edu.ie3.osmogrid.guardian.OsmoGridGuardian.RunGuardianDied +import edu.ie3.osmogrid.guardian.OsmoGridGuardian.{ + GuardianData, + RunGuardianDied +} import edu.ie3.osmogrid.guardian.run.RunGuardian import edu.ie3.osmogrid.guardian.run.Request import edu.ie3.osmogrid.io.output.ResultListener.ResultEvent @@ -53,5 +56,25 @@ class OsmoGridGuardianSpec extends UnitSpec { ) } } + + "checking for state data" should { + "bring up empty data" in { + GuardianData.empty shouldBe GuardianData(Seq.empty[UUID]) + } + + val run0 = UUID.randomUUID() + val run1 = UUID.randomUUID() + "properly add a new run to existing data" in { + GuardianData(Seq(run0)) + .append(run1) + .runs should contain theSameElementsAs Seq(run0, run1) + } + + "properly remove a run from existing data" in { + GuardianData(Seq(run0, run1)) + .remove(run0) + .runs should contain theSameElementsAs Seq(run1) + } + } } } From 5a7524cd1d85e4a5576c4ddc654ded22872f3e0f Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 24 Jan 2022 13:59:58 +0100 Subject: [PATCH 29/38] Check RunGuardian in idle --- .../osmogrid/guardian/run/RunSupport.scala | 75 ++++++------- src/test/resources/testConfig.conf | 2 +- .../guardian/run/RunGuardianSpec.scala | 100 ++++++++++++++++++ 3 files changed, 140 insertions(+), 37 deletions(-) create mode 100644 src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala index 63f2f5e4..e39e9fc5 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala @@ -36,48 +36,51 @@ trait RunSupport { ctx: ActorContext[Request] ): Try[ChildReferences] = { val log = ctx.log - ConfigFailFast.check( - runGuardianData.cfg, - runGuardianData.additionalListener - ) - log.info( - s"Initializing grid generation for run with id '${runGuardianData.runId}'!" - ) + Try { + ConfigFailFast.check( + runGuardianData.cfg, + runGuardianData.additionalListener + ) + }.flatMap { validConfig => + log.info( + s"Initializing grid generation for run with id '${runGuardianData.runId}'!" + ) - /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ - runGuardianData.cfg.generation match { - case Generation(Some(lvConfig)) => - ctx.log.info("Starting low voltage grid coordinator ...") - val (inputProvider, resultEventListener) = - spawnIoActors( + /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ + runGuardianData.cfg.generation match { + case Generation(Some(lvConfig)) => + ctx.log.info("Starting low voltage grid coordinator ...") + val (inputProvider, resultEventListener) = + spawnIoActors( + runGuardianData.runId, + runGuardianData.cfg.input, + runGuardianData.cfg.output, + ctx + ) + val lvCoordinator = startLvGridGeneration( runGuardianData.runId, - runGuardianData.cfg.input, - runGuardianData.cfg.output, + lvConfig, + runGuardianData.msgAdapters.lvCoordinator, ctx ) - val lvCoordinator = startLvGridGeneration( - runGuardianData.runId, - lvConfig, - runGuardianData.msgAdapters.lvCoordinator, - ctx - ) - Success( - ChildReferences( - inputProvider, - resultEventListener, - runGuardianData.additionalListener, - Some(lvCoordinator) + Success( + ChildReferences( + inputProvider, + resultEventListener, + runGuardianData.additionalListener, + Some(lvCoordinator) + ) ) - ) - case unsupported => - ctx.log.error( - s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${runGuardianData.runId}'!" - ) - Failure( - UnsupportedRequestException( - s"Unable to issue a generation run with the given parameters: '${runGuardianData.cfg.generation}'" + case unsupported => + ctx.log.error( + s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${runGuardianData.runId}'!" ) - ) + Failure( + UnsupportedRequestException( + s"Unable to issue a generation run with the given parameters: '${runGuardianData.cfg.generation}'" + ) + ) + } } } diff --git a/src/test/resources/testConfig.conf b/src/test/resources/testConfig.conf index 7f5c378e..f265eaa1 100644 --- a/src/test/resources/testConfig.conf +++ b/src/test/resources/testConfig.conf @@ -1,5 +1,5 @@ input.osm.pbf.file = "input_file_path" input.asset.file.directory = "asset_directory" input.asset.file.hierarchic = false -output.file.directory = "output_file_path" +output.csv.directory = "output_file_path" generation.lv.distinctHouseConnections = true diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala new file mode 100644 index 00000000..2d1423eb --- /dev/null +++ b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala @@ -0,0 +1,100 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian.run + +import akka.actor.testkit.typed.CapturedLogEvent +import akka.actor.testkit.typed.Effect.{MessageAdapter, Spawned, WatchedWith} +import akka.actor.testkit.typed.scaladsl.BehaviorTestKit +import akka.actor.typed.{ActorRef, Behavior} +import edu.ie3.osmogrid.cfg.OsmoGridConfigFactory +import edu.ie3.osmogrid.exception.IllegalConfigException +import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener.ResultEvent +import edu.ie3.osmogrid.lv.LvCoordinator +import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids +import edu.ie3.test.common.UnitSpec +import org.slf4j.event.Level + +import java.util.UUID + +class RunGuardianSpec extends UnitSpec { + "Having a run guardian" when { + val runId = UUID.randomUUID() + val validConfig = OsmoGridConfigFactory.defaultTestConfig + + "being idle" should { + val idleTestKit = BehaviorTestKit( + RunGuardian(validConfig, Seq.empty[ActorRef[ResultEvent]], runId) + ) + + "log an error if faced to not supported request" in { + idleTestKit.run(ResultEventListenerDied) + + idleTestKit.logEntries() should contain only CapturedLogEvent( + Level.ERROR, + s"Received a message, that I don't understand during idle phase of run $runId.\n\tMessage: $ResultEventListenerDied" + ) + } + + "log an error, if initiation of a run is impossible" in { + val maliciousConfig = OsmoGridConfigFactory + .parseWithoutFallback("") + .getOrElse(fail("Unable to parse malicious config")) + val maliciousIdleTestKit = BehaviorTestKit( + RunGuardian(maliciousConfig, Seq.empty[ActorRef[ResultEvent]], runId) + ) + + maliciousIdleTestKit.run(Run) + maliciousIdleTestKit.logEntries() should contain only CapturedLogEvent( + Level.ERROR, + s"Unable to start run $runId.", + Some( + IllegalConfigException( + "You have to provide at least one input data type for asset information!" + ) + ), + None + ) + } + + "spawns children, if input is fine" in { + idleTestKit.run(Run) + + /* Two message adapters are registered */ + idleTestKit + .expectEffectType[MessageAdapter[LvCoordinator.Response, Request]] + idleTestKit.expectEffectType[MessageAdapter[ResultEvent, Request]] + + /* Check if I/O actors and LvCoordinator are spawned and watched correctly */ + idleTestKit.expectEffectPF { + case Spawned(_: Behavior[InputDataProvider.Request], name, props) => + name shouldBe s"InputDataProvider_$runId" + } + idleTestKit + .expectEffectType[WatchedWith[InputDataProvider.Request, Watch]] + idleTestKit.expectEffectPF { + case Spawned(_: Behavior[ResultEvent], name, props) => + name shouldBe s"PersistenceResultListener_$runId" + } + idleTestKit.expectEffectType[WatchedWith[ResultEvent, Watch]] + idleTestKit.expectEffectPF { + case Spawned(_: Behavior[LvCoordinator.Request], name, props) => + name shouldBe s"LvCoordinator_$runId" + } + idleTestKit.expectEffectType[WatchedWith[LvCoordinator.Request, Watch]] + + /* Check for child messages */ + idleTestKit + .childInbox[LvCoordinator.Request](s"LvCoordinator_$runId") + .receiveAll() + .exists { case ReqLvGrids(cfg, replyTo) => + validConfig.generation.lv.contains(cfg) + } shouldBe true + } + } + } +} From b71e92e612441d9bad8abc85e5a24f53706d17c6 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 24 Jan 2022 14:34:42 +0100 Subject: [PATCH 30/38] Start testing the running state --- .../osmogrid/guardian/run/RunGuardian.scala | 2 +- .../guardian/run/RunGuardianSpec.scala | 51 +++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala index 7e21085d..15268fd2 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala @@ -96,7 +96,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { * @return * The next state */ - private def running( + protected def running( runGuardianData: RunGuardianData, childReferences: ChildReferences ): Behavior[Request] = Behaviors.receive { diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala index 2d1423eb..3732f3c9 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala @@ -8,11 +8,15 @@ package edu.ie3.osmogrid.guardian.run import akka.actor.testkit.typed.CapturedLogEvent import akka.actor.testkit.typed.Effect.{MessageAdapter, Spawned, WatchedWith} -import akka.actor.testkit.typed.scaladsl.BehaviorTestKit +import akka.actor.testkit.typed.scaladsl.{ + BehaviorTestKit, + ScalaTestWithActorTestKit +} import akka.actor.typed.{ActorRef, Behavior} import edu.ie3.osmogrid.cfg.OsmoGridConfigFactory import edu.ie3.osmogrid.exception.IllegalConfigException import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener import edu.ie3.osmogrid.io.output.ResultListener.ResultEvent import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids @@ -21,12 +25,12 @@ import org.slf4j.event.Level import java.util.UUID -class RunGuardianSpec extends UnitSpec { +class RunGuardianSpec extends ScalaTestWithActorTestKit with UnitSpec { "Having a run guardian" when { val runId = UUID.randomUUID() val validConfig = OsmoGridConfigFactory.defaultTestConfig - "being idle" should { + "being idle state" should { val idleTestKit = BehaviorTestKit( RunGuardian(validConfig, Seq.empty[ActorRef[ResultEvent]], runId) ) @@ -96,5 +100,46 @@ class RunGuardianSpec extends UnitSpec { } shouldBe true } } + + "being in running state" should { + val running = PrivateMethod[Behavior[Request]](Symbol("running")) + + /* Test probes */ + val lvCoordinatorAdapter = + testKit.createTestProbe[LvCoordinator.Response]() + val resultListenerAdapter = + testKit.createTestProbe[ResultListener.Response]() + val inputDataProvider = + testKit.createTestProbe[InputDataProvider.Request]() + val resultListener = testKit.createTestProbe[ResultEvent]() + val lvCoordinator = testKit.createTestProbe[LvCoordinator.Request]() + + /* State data */ + val runGuardianData = RunGuardianData( + runId, + validConfig, + Seq.empty[ActorRef[ResultEvent]], + MessageAdapters(lvCoordinatorAdapter.ref, resultListenerAdapter.ref) + ) + val childReferences = ChildReferences( + inputDataProvider.ref, + Some(resultListener.ref), + Seq.empty, + Some(lvCoordinator.ref) + ) + + val runningTestKit = BehaviorTestKit( + RunGuardian invokePrivate running(runGuardianData, childReferences) + ) + + "log an error if faced to not supported request" in { + runningTestKit.run(Run) + + runningTestKit.logEntries() should contain only CapturedLogEvent( + Level.ERROR, + s"Received a message, that I don't understand during active run $runId.\n\tMessage: $Run" + ) + } + } } } From efbfe4c4a6e0ccf1cbce3b9f4e758f4bf1e00fde Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 24 Jan 2022 15:01:21 +0100 Subject: [PATCH 31/38] Improve coordinated shutdown phase --- .../osmogrid/guardian/run/StopSupport.scala | 10 ++- .../osmogrid/io/input/InputDataProvider.scala | 12 +-- .../osmogrid/io/output/ResultListener.scala | 16 ++-- .../edu/ie3/osmogrid/lv/LvCoordinator.scala | 86 +++++++++++-------- .../guardian/run/RunGuardianSpec.scala | 16 ++++ 5 files changed, 88 insertions(+), 52 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala index ff41ffc3..a3ac72ff 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/StopSupport.scala @@ -6,8 +6,13 @@ package edu.ie3.osmogrid.guardian.run +import akka.actor.typed.SupervisorStrategy.Stop import akka.actor.typed.scaladsl.ActorContext import edu.ie3.osmogrid.guardian.run.RunGuardian +import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener +import edu.ie3.osmogrid.lv.LvCoordinator +import edu.ie3.osmogrid.lv.LvCoordinator.Terminate import java.util.UUID @@ -28,8 +33,9 @@ trait StopSupport { childReferences: ChildReferences, ctx: ActorContext[Request] ): StoppingData = { - ctx.stop(childReferences.inputDataProvider) - childReferences.resultListener.foreach(ctx.stop) + childReferences.lvCoordinator.foreach(_ ! LvCoordinator.Terminate) + childReferences.inputDataProvider ! InputDataProvider.Terminate + childReferences.resultListener.foreach(_ ! ResultListener.Terminate) StoppingData( runId, diff --git a/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala b/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala index edab8dfe..1a967ca5 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/input/InputDataProvider.scala @@ -18,8 +18,7 @@ object InputDataProvider { sealed trait Request final case class Read() extends Request // todo this read method should contain configuration parameters for the actual source + potential filter options - final case class Terminate(replyTo: ActorRef[OsmoGridGuardian.Request]) - extends Request + object Terminate extends Request sealed trait Response @@ -29,14 +28,15 @@ object InputDataProvider { case (ctx, _: Read) => ctx.log.warn("Reading of data not yet implemented.") Behaviors.same - case (ctx, Terminate(_)) => + case (ctx, Terminate) => ctx.log.info("Stopping input data provider") - // TODO: Any closing of sources and stuff - Behaviors.stopped + Behaviors.stopped { () => cleanUp() } } .receiveSignal { case (ctx, PostStop) => ctx.log.info("Requested to stop.") - // TODO: Any closing of sources and stuff + cleanUp() Behaviors.same } + + private def cleanUp(): Unit = ??? } diff --git a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala index 7f7690e1..07b7671f 100644 --- a/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala +++ b/src/main/scala/edu/ie3/osmogrid/io/output/ResultListener.scala @@ -24,6 +24,7 @@ object ResultListener { replyTo: ActorRef[Response] ) extends Request with ResultEvent + object Terminate extends Request with ResultEvent sealed trait Response final case class ResultHandled( @@ -36,16 +37,19 @@ object ResultListener { def apply(runId: UUID, cfg: OsmoGridConfig.Output): Behavior[ResultEvent] = Behaviors - .receive[ResultEvent] { case (ctx, GridResult(grid, replyTo)) => - ctx.log.info(s"Received grid result for run id '${runId.toString}'") - // TODO: Actual persistence and stuff, closing sinks, ... - replyTo ! ResultHandled(runId, ctx.self) - Behaviors.same + .receive[ResultEvent] { + case (ctx, GridResult(grid, replyTo)) => + ctx.log.info(s"Received grid result for run id '${runId.toString}'") + // TODO: Actual persistence and stuff, ... + replyTo ! ResultHandled(runId, ctx.self) + Behaviors.stopped { () => cleanUp() } + case (ctx, Terminate) => Behaviors.stopped { () => cleanUp() } } .receiveSignal { case (ctx, PostStop) => ctx.log.info("Requested to stop.") - // TODO: Any closing of sources and stuff + cleanUp() Behaviors.same } + private def cleanUp(): Unit = ??? } diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala index 61245a85..7cbdfd49 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala @@ -6,7 +6,7 @@ package edu.ie3.osmogrid.lv -import akka.actor.typed.{ActorRef, Behavior, SupervisorStrategy} +import akka.actor.typed.{ActorRef, Behavior, PostStop, SupervisorStrategy} import akka.actor.typed.scaladsl.{Behaviors, Routers} import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.osmogrid.cfg.OsmoGridConfig @@ -22,54 +22,64 @@ object LvCoordinator { replyTo: ActorRef[Response] ) extends Request + object Terminate extends Request + sealed trait Response final case class RepLvGrids(grids: Seq[SubGridContainer]) extends Response def apply(): Behavior[Request] = idle - private def idle: Behavior[Request] = Behaviors.receive { (ctx, msg) => - msg match { - case ReqLvGrids( - Lv( - amountOfGridGenerators, - amountOfRegionCoordinators, - distinctHouseConnections - ), - replyTo - ) => - ctx.log.info("Starting generation of low voltage grids!") - /* TODO: + private def idle: Behavior[Request] = Behaviors + .receive[Request] { (ctx, msg) => + msg match { + case ReqLvGrids( + Lv( + amountOfGridGenerators, + amountOfRegionCoordinators, + distinctHouseConnections + ), + replyTo + ) => + ctx.log.info("Starting generation of low voltage grids!") + /* TODO: 1) Ask for OSM data 2) Ask for asset data 3) Split up osm data at municipality boundaries 4) start generation */ - /* Spawn a pool of workers to build grids from sub-graphs */ - val lvGeneratorPool = - Routers.pool(poolSize = amountOfGridGenerators) { - // Restart workers on failure - Behaviors - .supervise(LvGenerator()) - .onFailure(SupervisorStrategy.restart) - } - val lvGeneratorProxy = ctx.spawn(lvGeneratorPool, "LvGeneratorPool") + /* Spawn a pool of workers to build grids from sub-graphs */ + val lvGeneratorPool = + Routers.pool(poolSize = amountOfGridGenerators) { + // Restart workers on failure + Behaviors + .supervise(LvGenerator()) + .onFailure(SupervisorStrategy.restart) + } + val lvGeneratorProxy = ctx.spawn(lvGeneratorPool, "LvGeneratorPool") - /* Spawn a pool of workers to build grids for one municipality */ - val lvRegionCoordinatorPool = - Routers.pool(poolSize = amountOfRegionCoordinators) { - // Restart workers on failure - Behaviors - .supervise(LvRegionCoordinator(lvGeneratorProxy)) - .onFailure(SupervisorStrategy.restart) - } - val lvRegionCoordinatorProxy = - ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") + /* Spawn a pool of workers to build grids for one municipality */ + val lvRegionCoordinatorPool = + Routers.pool(poolSize = amountOfRegionCoordinators) { + // Restart workers on failure + Behaviors + .supervise(LvRegionCoordinator(lvGeneratorProxy)) + .onFailure(SupervisorStrategy.restart) + } + val lvRegionCoordinatorProxy = + ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") - replyTo ! RepLvGrids(Seq.empty[SubGridContainer]) - Behaviors.stopped - case unsupported => - ctx.log.error(s"Received unsupported message: $unsupported") - Behaviors.stopped + replyTo ! RepLvGrids(Seq.empty[SubGridContainer]) + Behaviors.stopped { () => cleanUp() } + case unsupported => + ctx.log.error(s"Received unsupported message: $unsupported") + Behaviors.stopped { () => cleanUp() } + } + } + .receiveSignal { case (ctx, PostStop) => + ctx.log.info("Got terminated by ActorSystem.") + cleanUp() + Behaviors.same } - } + + private def cleanUp(): Unit = ??? } diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala index 3732f3c9..b2a4b2f0 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala @@ -140,6 +140,22 @@ class RunGuardianSpec extends ScalaTestWithActorTestKit with UnitSpec { s"Received a message, that I don't understand during active run $runId.\n\tMessage: $Run" ) } + + "initiate coordinated shutdown, if somebody unexpectedly dies" in { + runningTestKit.run(LvCoordinatorDied) + + /* Event is logged */ + runningTestKit.logEntries() should contain( + CapturedLogEvent( + Level.WARN, + s"Lv coordinator for run $runId unexpectedly died. Start coordinated shut down phase for this run." + ) + ) + /* All children are sent a termination request */ + lvCoordinator.expectMessage(LvCoordinator.Terminate) + inputDataProvider.expectMessage(InputDataProvider.Terminate) + resultListener.expectMessage(ResultListener.Terminate) + } } } } From 056caa0f7cac7a6ceed3176e74a9c02071cd8e46 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Mon, 24 Jan 2022 15:14:55 +0100 Subject: [PATCH 32/38] Test receiving results --- .../guardian/run/RunGuardianSpec.scala | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala index b2a4b2f0..c84c3422 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala @@ -13,14 +13,15 @@ import akka.actor.testkit.typed.scaladsl.{ ScalaTestWithActorTestKit } import akka.actor.typed.{ActorRef, Behavior} +import edu.ie3.datamodel.models.input.container.SubGridContainer import edu.ie3.osmogrid.cfg.OsmoGridConfigFactory import edu.ie3.osmogrid.exception.IllegalConfigException import edu.ie3.osmogrid.io.input.InputDataProvider import edu.ie3.osmogrid.io.output.ResultListener -import edu.ie3.osmogrid.io.output.ResultListener.ResultEvent +import edu.ie3.osmogrid.io.output.ResultListener.{GridResult, ResultEvent} import edu.ie3.osmogrid.lv.LvCoordinator import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids -import edu.ie3.test.common.UnitSpec +import edu.ie3.test.common.{GridSupport, UnitSpec} import org.slf4j.event.Level import java.util.UUID @@ -95,8 +96,10 @@ class RunGuardianSpec extends ScalaTestWithActorTestKit with UnitSpec { idleTestKit .childInbox[LvCoordinator.Request](s"LvCoordinator_$runId") .receiveAll() - .exists { case ReqLvGrids(cfg, replyTo) => - validConfig.generation.lv.contains(cfg) + .exists { + case ReqLvGrids(cfg, replyTo) => + validConfig.generation.lv.contains(cfg) + case _ => false } shouldBe true } } @@ -141,6 +144,26 @@ class RunGuardianSpec extends ScalaTestWithActorTestKit with UnitSpec { ) } + "handles an incoming result" in new GridSupport { + runningTestKit.run( + MessageAdapters.WrappedLvCoordinatorResponse( + LvCoordinator.RepLvGrids(Seq(mockSubGrid(1))) + ) + ) + + /* Event is logged */ + runningTestKit.logEntries() should contain allOf (CapturedLogEvent( + Level.INFO, + "All lv grids successfully generated." + ), CapturedLogEvent( + Level.DEBUG, + "No further generation steps intended. Hand over results to result handler." + )) + + /* Result is forwarded to listener */ + resultListener.expectMessageType[GridResult] + } + "initiate coordinated shutdown, if somebody unexpectedly dies" in { runningTestKit.run(LvCoordinatorDied) From 67254c6b7a2437b3281aa2a6cd56d2aa9d936722 Mon Sep 17 00:00:00 2001 From: Sebastian Peter <14994800+sebastian-peter@users.noreply.github.com> Date: Mon, 24 Jan 2022 15:42:27 +0100 Subject: [PATCH 33/38] Adding tests for RunGuardian.stopping --- .../ie3/osmogrid/guardian/run/package.scala | 2 +- .../guardian/run/RunGuardianSpec.scala | 63 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala index 981c878f..22162c32 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala @@ -93,6 +93,6 @@ package object run { ) extends StateData { def allChildrenTerminated: Boolean = inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated - .contains(true) + .forall(x => x) } } diff --git a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala index c84c3422..d163be14 100644 --- a/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/guardian/run/RunGuardianSpec.scala @@ -180,5 +180,68 @@ class RunGuardianSpec extends ScalaTestWithActorTestKit with UnitSpec { resultListener.expectMessage(ResultListener.Terminate) } } + + "being in stopping state without a LvCoordinator" should { + val stopping = PrivateMethod[Behavior[Request]](Symbol("stopping")) + + val stoppingData = StoppingData( + runId, + inputDataProviderTerminated = false, + resultListenerTerminated = false, + lvCoordinatorTerminated = None + ) + + val stoppingTestKit = BehaviorTestKit( + RunGuardian invokePrivate stopping(stoppingData) + ) + + "log an error if faced to not supported request" in { + stoppingTestKit.run(Run) + + stoppingTestKit.logEntries() should contain only CapturedLogEvent( + Level.ERROR, + s"Received a message, that I don't understand during coordinated shutdown phase of run $runId.\n\tMessage: $Run" + ) + } + + "stop itself only once all awaited termination messages have been received" in { + stoppingTestKit.run(ResultEventListenerDied) + + stoppingTestKit.isAlive shouldBe true + + stoppingTestKit.run(InputDataProviderDied) + + stoppingTestKit.isAlive shouldBe false + } + } + + "being in stopping state with a LvCoordinator" should { + val stopping = PrivateMethod[Behavior[Request]](Symbol("stopping")) + + val stoppingData = StoppingData( + runId, + inputDataProviderTerminated = false, + resultListenerTerminated = false, + lvCoordinatorTerminated = Some(false) + ) + + val stoppingTestKit = BehaviorTestKit( + RunGuardian invokePrivate stopping(stoppingData) + ) + + "stop itself only once all awaited termination messages have been received" in { + stoppingTestKit.run(InputDataProviderDied) + + stoppingTestKit.isAlive shouldBe true + + stoppingTestKit.run(ResultEventListenerDied) + + stoppingTestKit.isAlive shouldBe true + + stoppingTestKit.run(LvCoordinatorDied) + + stoppingTestKit.isAlive shouldBe false + } + } } } From b8a54e7d638193ab2ace6afc844625489abd8ce8 Mon Sep 17 00:00:00 2001 From: Sebastian Peter <14994800+sebastian-peter@users.noreply.github.com> Date: Mon, 24 Jan 2022 15:51:12 +0100 Subject: [PATCH 34/38] Preempting @johanneshiry's review comment --- src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala index 22162c32..22eb0985 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala @@ -93,6 +93,6 @@ package object run { ) extends StateData { def allChildrenTerminated: Boolean = inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated - .forall(x => x) + .forall(terminated => terminated) } } From c72c26c7d45c41f925c564a0772742250afa31c2 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 27 Jan 2022 11:31:07 +0100 Subject: [PATCH 35/38] Get rid of package objects --- .../ie3/osmogrid/guardian/run/package.scala | 98 ------------------- .../edu/ie3/osmogrid/guardian/run/run.scala | 98 +++++++++++++++++++ 2 files changed, 98 insertions(+), 98 deletions(-) delete mode 100644 src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala create mode 100644 src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala deleted file mode 100644 index 981c878f..00000000 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/package.scala +++ /dev/null @@ -1,98 +0,0 @@ -/* - * © 2022. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ - -package edu.ie3.osmogrid.guardian - -import akka.actor.typed.ActorRef -import edu.ie3.osmogrid.cfg.OsmoGridConfig -import edu.ie3.osmogrid.io.input.InputDataProvider -import edu.ie3.osmogrid.io.output.ResultListener -import edu.ie3.osmogrid.lv.LvCoordinator - -import java.util.UUID - -package object run { - sealed trait Request - - object Run extends Request - - /** Container object with all available adapters for outside protocol messages - * - * @param lvCoordinator - * Adapter for messages from [[LvCoordinator]] - * @param resultListener - * Adapter for messages from [[ResultEventListener]] - */ - private final case class MessageAdapters( - lvCoordinator: ActorRef[LvCoordinator.Response], - resultListener: ActorRef[ResultListener.Response] - ) - - private object MessageAdapters { - final case class WrappedLvCoordinatorResponse( - response: LvCoordinator.Response - ) extends Request - - final case class WrappedListenerResponse( - response: ResultListener.Response - ) extends Request - } - - sealed trait Response - - final case class Done(runId: UUID) extends Response - - sealed trait Watch extends Request - - private object InputDataProviderDied extends Watch - - private object ResultEventListenerDied extends Watch - - private object LvCoordinatorDied extends Watch - - private final case class ChildReferences( - inputDataProvider: ActorRef[InputDataProvider.Request], - resultListener: Option[ActorRef[ResultListener.ResultEvent]], - additionalResultListeners: Seq[ActorRef[ResultListener.ResultEvent]], - lvCoordinator: Option[ActorRef[LvCoordinator.Request]] - ) { - def resultListeners: Seq[ActorRef[ResultListener.ResultEvent]] = - resultListener - .map(Seq(_)) - .getOrElse(Seq.empty) ++ additionalResultListeners - } - - private sealed trait StateData - private final case class RunGuardianData( - runId: UUID, - cfg: OsmoGridConfig, - additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], - msgAdapters: MessageAdapters - ) extends StateData - - /** Meta data to keep track of which children already terminated during the - * coordinated shutdown phase - * - * @param runId - * Identifier of the run - * @param inputDataProviderTerminated - * If the [[InputDataProvider]] has stopped - * @param resultListenerTerminated - * If the [[ResultListener]] has stopped - * @param lvCoordinatorTerminated - * Optional information, if the [[LvCoordinator]] has stopped - */ - private final case class StoppingData( - runId: UUID, - inputDataProviderTerminated: Boolean, - resultListenerTerminated: Boolean, - lvCoordinatorTerminated: Option[Boolean] - ) extends StateData { - def allChildrenTerminated: Boolean = - inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated - .contains(true) - } -} diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala new file mode 100644 index 00000000..ac1bb7f3 --- /dev/null +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala @@ -0,0 +1,98 @@ +/* + * © 2022. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ + +package edu.ie3.osmogrid.guardian.run + +import akka.actor.typed.ActorRef +import edu.ie3.osmogrid.cfg.OsmoGridConfig +import edu.ie3.osmogrid.io.input.InputDataProvider +import edu.ie3.osmogrid.io.output.ResultListener +import edu.ie3.osmogrid.lv.LvCoordinator + +import java.util.UUID + +/* This file only contains package-level definitions */ + +sealed trait Request + +object Run extends Request + +/** Container object with all available adapters for outside protocol messages + * + * @param lvCoordinator + * Adapter for messages from [[LvCoordinator]] + * @param resultListener + * Adapter for messages from [[ResultEventListener]] + */ +private final case class MessageAdapters( + lvCoordinator: ActorRef[LvCoordinator.Response], + resultListener: ActorRef[ResultListener.Response] +) + +private object MessageAdapters { + final case class WrappedLvCoordinatorResponse( + response: LvCoordinator.Response + ) extends Request + + final case class WrappedListenerResponse( + response: ResultListener.Response + ) extends Request +} + +sealed trait Response + +final case class Done(runId: UUID) extends Response + +sealed trait Watch extends Request + +private object InputDataProviderDied extends Watch + +private object ResultEventListenerDied extends Watch + +private object LvCoordinatorDied extends Watch + +private final case class ChildReferences( + inputDataProvider: ActorRef[InputDataProvider.Request], + resultListener: Option[ActorRef[ResultListener.ResultEvent]], + additionalResultListeners: Seq[ActorRef[ResultListener.ResultEvent]], + lvCoordinator: Option[ActorRef[LvCoordinator.Request]] +) { + def resultListeners: Seq[ActorRef[ResultListener.ResultEvent]] = + resultListener + .map(Seq(_)) + .getOrElse(Seq.empty) ++ additionalResultListeners +} + +private sealed trait StateData +private final case class RunGuardianData( + runId: UUID, + cfg: OsmoGridConfig, + additionalListener: Seq[ActorRef[ResultListener.ResultEvent]], + msgAdapters: MessageAdapters +) extends StateData + +/** Meta data to keep track of which children already terminated during the + * coordinated shutdown phase + * + * @param runId + * Identifier of the run + * @param inputDataProviderTerminated + * If the [[InputDataProvider]] has stopped + * @param resultListenerTerminated + * If the [[ResultListener]] has stopped + * @param lvCoordinatorTerminated + * Optional information, if the [[LvCoordinator]] has stopped + */ +private final case class StoppingData( + runId: UUID, + inputDataProviderTerminated: Boolean, + resultListenerTerminated: Boolean, + lvCoordinatorTerminated: Option[Boolean] +) extends StateData { + def allChildrenTerminated: Boolean = + inputDataProviderTerminated && resultListenerTerminated && lvCoordinatorTerminated + .contains(true) +} From 036e87401b4413affc48231aa923ce22ea32e5a8 Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 27 Jan 2022 11:46:52 +0100 Subject: [PATCH 36/38] Address reviewer's comments --- .../edu/ie3/osmogrid/cfg/ConfigFailFast.scala | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala index 46150bf0..7f5c9604 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala @@ -73,12 +73,8 @@ object ConfigFailFast extends LazyLogging { } private def checkAssetInputFile(file: OsmoGridConfig.Input.Asset.File): Unit = - file match { - case File(directory, _) if directory.isEmpty => - throw IllegalConfigException("Asset input directory may be set!") - case _: File => - /* Everything is fine */ - } + if (file.directory.isEmpty) + throw IllegalConfigException("Asset input directory may be set!") private def checkOsmInputConfig(osm: OsmoGridConfig.Input.Osm): Unit = osm match { @@ -90,12 +86,7 @@ object ConfigFailFast extends LazyLogging { } private def checkPbfFileDefinition(pbf: OsmoGridConfig.Input.Osm.Pbf): Unit = - pbf match { - case OsmoGridConfig.Input.Osm.Pbf(file) if file.isEmpty => - throw IllegalConfigException("Pbf file may be set!") - case OsmoGridConfig.Input.Osm.Pbf(_) => - /* Everything is fine */ - } + if (pbf.file.isEmpty) throw IllegalConfigException("Pbf file may be set!") private def checkOutputConfig( output: OsmoGridConfig.Output, From 6c0807b6626c72ba165d86becab8da890125011f Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 27 Jan 2022 12:45:41 +0100 Subject: [PATCH 37/38] Making death watch events public --- .../scala/edu/ie3/osmogrid/guardian/run/run.scala | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala index e113bddc..1ad53252 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/run.scala @@ -16,6 +16,7 @@ import java.util.UUID /* This file only contains package-level definitions */ +/* Received requests */ sealed trait Request object Run extends Request @@ -42,17 +43,19 @@ private object MessageAdapters { ) extends Request } -sealed trait Response +/* Death watch messages */ +sealed trait Watch extends Request -final case class Done(runId: UUID) extends Response +object InputDataProviderDied extends Watch -sealed trait Watch extends Request +object ResultEventListenerDied extends Watch -private object InputDataProviderDied extends Watch +object LvCoordinatorDied extends Watch -private object ResultEventListenerDied extends Watch +/* Sent out responses */ +sealed trait Response -private object LvCoordinatorDied extends Watch +final case class Done(runId: UUID) extends Response private final case class ChildReferences( inputDataProvider: ActorRef[InputDataProvider.Request], From 551ffadd9a32f31b7cabd09b24d2c8d58d1261da Mon Sep 17 00:00:00 2001 From: "Kittl, Chris" Date: Thu, 27 Jan 2022 17:39:04 +0100 Subject: [PATCH 38/38] Addressing reviewer's comments --- .../edu/ie3/osmogrid/cfg/ConfigFailFast.scala | 31 +++---- .../osmogrid/guardian/run/RunGuardian.scala | 2 +- .../osmogrid/guardian/run/RunSupport.scala | 74 ++++++++-------- .../guardian/run/SubGridHandling.scala | 1 - .../edu/ie3/osmogrid/lv/LvCoordinator.scala | 6 +- .../ie3/osmogrid/lv/LvRegionCoordinator.scala | 9 +- .../ie3/osmogrid/cfg/ConfigFailFastSpec.scala | 86 ++++++++++++------- 7 files changed, 113 insertions(+), 96 deletions(-) diff --git a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala index 7f5c9604..fb11b7f0 100644 --- a/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala +++ b/src/main/scala/edu/ie3/osmogrid/cfg/ConfigFailFast.scala @@ -16,15 +16,20 @@ import edu.ie3.osmogrid.cfg.OsmoGridConfig.Input.Asset.File import edu.ie3.osmogrid.exception.IllegalConfigException import edu.ie3.osmogrid.io.output.ResultListener +import scala.util.Try + object ConfigFailFast extends LazyLogging { def check( cfg: OsmoGridConfig, additionalListener: Seq[ActorRef[ResultListener.ResultEvent]] = Seq.empty - ): Unit = cfg match { - case OsmoGridConfig(generation, input, output) => - checkInputConfig(input) - checkOutputConfig(output, additionalListener) - checkGenerationConfig(generation) + ): Try[OsmoGridConfig] = Try { + cfg match { + case OsmoGridConfig(generation, input, output) => + checkInputConfig(input) + checkOutputConfig(output, additionalListener) + checkGenerationConfig(generation) + } + cfg } private def checkGenerationConfig(generation: Generation): Unit = @@ -105,14 +110,10 @@ object ConfigFailFast extends LazyLogging { ) } - private def checkOutputFile(file: OsmoGridConfig.Output.Csv): Unit = - file match { - case OsmoGridConfig.Output.Csv(directory, _, separator) - if directory.isEmpty || separator.isEmpty => - throw IllegalConfigException( - "Output directory and separator must be set when using .csv file sink!" - ) - case _: OsmoGridConfig.Output.Csv => - /* Everything is fine */ - } + private def checkOutputFile(file: OsmoGridConfig.Output.Csv): Unit = if ( + file.directory.isEmpty || file.separator.isEmpty + ) + throw IllegalConfigException( + "Output directory and separator must be set when using .csv file sink!" + ) } diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala index 15268fd2..7e21085d 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunGuardian.scala @@ -96,7 +96,7 @@ object RunGuardian extends RunSupport with StopSupport with SubGridHandling { * @return * The next state */ - protected def running( + private def running( runGuardianData: RunGuardianData, childReferences: ChildReferences ): Behavior[Request] = Behaviors.receive { diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala index e39e9fc5..5426832e 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/RunSupport.scala @@ -20,7 +20,7 @@ import edu.ie3.osmogrid.lv.LvCoordinator.ReqLvGrids import java.util.UUID import scala.util.{Failure, Success, Try} -trait RunSupport { +private trait RunSupport { /** Initiate a generation run and return the updated run meta data * @@ -36,52 +36,52 @@ trait RunSupport { ctx: ActorContext[Request] ): Try[ChildReferences] = { val log = ctx.log - Try { - ConfigFailFast.check( + ConfigFailFast + .check( runGuardianData.cfg, runGuardianData.additionalListener ) - }.flatMap { validConfig => - log.info( - s"Initializing grid generation for run with id '${runGuardianData.runId}'!" - ) + .flatMap { validConfig => + log.info( + s"Initializing grid generation for run with id '${runGuardianData.runId}'!" + ) - /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ - runGuardianData.cfg.generation match { - case Generation(Some(lvConfig)) => - ctx.log.info("Starting low voltage grid coordinator ...") - val (inputProvider, resultEventListener) = - spawnIoActors( + /* Check, which voltage level configs are given. Start with lv level, if this is desired for. */ + validConfig.generation match { + case Generation(Some(lvConfig)) => + ctx.log.info("Starting low voltage grid coordinator ...") + val (inputProvider, resultEventListener) = + spawnIoActors( + runGuardianData.runId, + validConfig.input, + validConfig.output, + ctx + ) + val lvCoordinator = startLvGridGeneration( runGuardianData.runId, - runGuardianData.cfg.input, - runGuardianData.cfg.output, + lvConfig, + runGuardianData.msgAdapters.lvCoordinator, ctx ) - val lvCoordinator = startLvGridGeneration( - runGuardianData.runId, - lvConfig, - runGuardianData.msgAdapters.lvCoordinator, - ctx - ) - Success( - ChildReferences( - inputProvider, - resultEventListener, - runGuardianData.additionalListener, - Some(lvCoordinator) + Success( + ChildReferences( + inputProvider, + resultEventListener, + runGuardianData.additionalListener, + Some(lvCoordinator) + ) ) - ) - case unsupported => - ctx.log.error( - s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${runGuardianData.runId}'!" - ) - Failure( - UnsupportedRequestException( - s"Unable to issue a generation run with the given parameters: '${runGuardianData.cfg.generation}'" + case unsupported => + ctx.log.error( + s"Received unsupported grid generation config '$unsupported'. Stopping run with id '${runGuardianData.runId}'!" ) - ) + Failure( + UnsupportedRequestException( + s"Unable to issue a generation run with the given parameters: '${validConfig.generation}'" + ) + ) + } } - } } /** Spawns both the input and the output actor for the given specific run diff --git a/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala b/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala index 1bba81d2..1c120bda 100644 --- a/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala +++ b/src/main/scala/edu/ie3/osmogrid/guardian/run/SubGridHandling.scala @@ -40,7 +40,6 @@ trait SubGridHandling { log.info("All lv grids successfully generated.") val updatedSubGrids = assignSubnetNumbers(grids) - // TODO: Check for mv config and issue run there, if applicable log.debug( "No further generation steps intended. Hand over results to result handler." ) diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala index 7cbdfd49..af6096bf 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvCoordinator.scala @@ -69,10 +69,10 @@ object LvCoordinator { ctx.spawn(lvRegionCoordinatorPool, "LvRegionCoordinatorPool") replyTo ! RepLvGrids(Seq.empty[SubGridContainer]) - Behaviors.stopped { () => cleanUp() } + Behaviors.stopped(cleanUp()) case unsupported => ctx.log.error(s"Received unsupported message: $unsupported") - Behaviors.stopped { () => cleanUp() } + Behaviors.stopped(cleanUp()) } } .receiveSignal { case (ctx, PostStop) => @@ -81,5 +81,5 @@ object LvCoordinator { Behaviors.same } - private def cleanUp(): Unit = ??? + private def cleanUp(): () => Unit = ??? } diff --git a/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala b/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala index 25b53b8e..0494ab69 100644 --- a/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala +++ b/src/main/scala/edu/ie3/osmogrid/lv/LvRegionCoordinator.scala @@ -15,12 +15,5 @@ object LvRegionCoordinator { def apply( lvGeneratorPool: ActorRef[LvGenerator.Request] - ): Behaviors.Receive[Request] = idle(lvGeneratorPool) - - private def idle( - lvGeneratorPool: ActorRef[LvGenerator.Request] - ): Behaviors.Receive[Request] = Behaviors.receive { case (ctx, unsupported) => - ctx.log.warn(s"Received unsupported message '$unsupported'.") - Behaviors.stopped - } + ): Behaviors.Receive[Request] = ??? } diff --git a/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala b/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala index 97f78401..76141007 100644 --- a/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala +++ b/src/test/scala/edu/ie3/osmogrid/cfg/ConfigFailFastSpec.scala @@ -24,9 +24,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = true""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "You have to provide at least one input data type for open street map information!" + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "You have to provide at least one input data type for open street map information!" + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -41,9 +44,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = true""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "Pbf file may be set!" + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "Pbf file may be set!" + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -56,9 +62,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = true""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "You have to provide at least one input data type for asset information!" + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "You have to provide at least one input data type for asset information!" + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -73,9 +82,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = true""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "Asset input directory may be set!" + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "Asset input directory may be set!" + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -91,9 +103,12 @@ class ConfigFailFastSpec extends UnitSpec { |output.csv.directory = "output_file_path"""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "At least one voltage level generation config has to be defined." + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "At least one voltage level generation config has to be defined." + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -112,9 +127,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = false""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "The amount of lv grid generation actors needs to be at least 1 (provided: 0)." + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "The amount of lv grid generation actors needs to be at least 1 (provided: 0)." + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -131,9 +149,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = false""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "The amount of lv grid generation actors needs to be at least 1 (provided: -42)." + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "The amount of lv grid generation actors needs to be at least 1 (provided: -42)." + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -150,9 +171,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = false""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "The amount of lv region coordination actors needs to be at least 1 (provided: 0)." + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "The amount of lv region coordination actors needs to be at least 1 (provided: 0)." + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -169,9 +193,12 @@ class ConfigFailFastSpec extends UnitSpec { |generation.lv.distinctHouseConnections = false""".stripMargin } match { case Success(cfg) => - val exc = - intercept[IllegalConfigException](ConfigFailFast.check(cfg)) - exc.msg shouldBe "The amount of lv region coordination actors needs to be at least 1 (provided: -42)." + ConfigFailFast.check(cfg) match { + case Failure(exception) => + exception.getMessage shouldBe "The amount of lv region coordination actors needs to be at least 1 (provided: -42)." + case Success(_) => + fail("Config check succeeded, but was meant to fail.") + } case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") } @@ -187,10 +214,7 @@ class ConfigFailFastSpec extends UnitSpec { |output.csv.directory = "output_file_path" |generation.lv.distinctHouseConnections = true""".stripMargin } match { - case Success(cfg) => - noException shouldBe thrownBy { - ConfigFailFast.check(cfg) - } + case Success(cfg) => ConfigFailFast.check(cfg) shouldBe Success(cfg) case Failure(exception) => fail(s"Config generation failed with an exception: '$exception'") }