diff --git a/README.md b/README.md index 88aa652a..d3ef09dd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # simBench2pdsm [![Build Status](https://travis-ci.org/ie3-institute/simbench4ie3.svg?branch=master)](https://travis-ci.org/ie3-institute/simbench4ie3) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/c339035212004716bab94c774da476aa)](https://app.codacy.com/gh/ie3-institute/simbench4ie3?utm_source=github.com&utm_medium=referral&utm_content=ie3-institute/simbench4ie3&utm_campaign=Badge_Grade_Dashboard) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/793affe18cd44718a66b07b2a7c45258)](https://www.codacy.com/gh/ie3-institute/simBench2psdm/dashboard?utm_source=github.com&utm_medium=referral&utm_content=ie3-institute/simBench2psdm&utm_campaign=Badge_Grade) [![codecov](https://codecov.io/gh/ie3-institute/simBench2psdm/branch/master/graph/badge.svg)](https://codecov.io/gh/ie3-institute/simBench2psdm) [![License](https://img.shields.io/github/license/ie3-institute/simbench4ie3)](https://github.com/ie3-institute/simbench4ie3/blob/master/LICENSE) diff --git a/build.gradle b/build.gradle index fb220031..ae7c5ade 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ ext { } group = 'com.github.ie3-institute' -description = 'simbench4ie3' +description = 'simbench2psdm' version = '1.1-SNAPSHOT' sourceCompatibility = javaVersion targetCompatibility = javaVersion @@ -48,20 +48,22 @@ dependencies { /* PowerSystemDataModel */ implementation ('com.github.ie3-institute:PowerSystemDataModel:2.1-SNAPSHOT') { /* Exclude nested logging and ie3 related dependencies */ - exclude group: 'org.slf4j', module: 'slf4j-api' + exclude group: 'org.slf4j' exclude group: 'com.github.ie3-institute' } /* util functions */ - implementation ('com.github.ie3-institute:PowerSystemUtils:1.5.3') { + implementation ('com.github.ie3-institute:PowerSystemUtils:1.6-SNAPSHOT') { /* Exclude nested logging and ie3 related dependencies */ - exclude group: 'org.slf4j', module: 'slf4j-api' + exclude group: 'org.slf4j' exclude group: 'com.github.ie3-institute' } implementation 'org.codehaus.groovy:groovy:3.0.8' implementation 'tech.units:indriya:2.1.2' + implementation 'org.locationtech.jts:jts-core:1.18.2' + implementation 'commons-io:commons-io:2.11.0' // logging implementation 'org.apache.logging.log4j:log4j-api:+' // log4j @@ -74,10 +76,11 @@ dependencies { // NEW scala libs // // CORE Scala // implementation "org.scala-lang:scala-library:$scalaBinaryVersion" + implementation 'org.scala-lang.modules:scala-parallel-collections_2.13:1.0.3' // TEST Scala // - testImplementation "org.scalatest:scalatest_${scalaVersion}:3.2.0" - testImplementation 'com.vladsch.flexmark:flexmark-all:0.35.10' + testImplementation "org.scalatest:scalatest_${scalaVersion}:3.2.10" + testImplementation 'com.vladsch.flexmark:flexmark-all:0.62.2' testImplementation "org.pegdown:pegdown:1.6.0" // HTML report for scalatest implementation 'org.mockito:mockito-core:3.11.2' // mocking framework diff --git a/docs/diagrams/RemoveSwitches.puml b/docs/diagrams/RemoveSwitches.puml new file mode 100644 index 00000000..3d0a7a31 --- /dev/null +++ b/docs/diagrams/RemoveSwitches.puml @@ -0,0 +1,18 @@ +@startuml + +(*) --> "Choose all closed switches" +--> "Identify switch groups" +note right: By iterating over all existing switches\nand building bags of nodes, that are\ndirectly connected via switches.If a node\n of a following switch is already in a bag,\nthen add the switch to thisswitch group. +--> "Take the next switch group" +--> "Identify main node" +note left: By counting the occurrences of\nthis node within all nodes of\nthat switch group. If all nodes\noccur only once, take the first\none. +--> "Build collapsing mapping" +note left: By assigning all nodes\nof a switch group to the\nmain node. +if "Last group" then + -->[true] "Consider mapping when converting node models" + --> "Do not convert switch models" + --> (*) +else + -->[false] "Take the next switch group" + +@enduml \ No newline at end of file diff --git a/docs/SimbenchDataModel.puml b/docs/diagrams/SimbenchDataModel.puml similarity index 100% rename from docs/SimbenchDataModel.puml rename to docs/diagrams/SimbenchDataModel.puml diff --git a/inputData/config/mvLvNoSwitches.conf b/inputData/config/mvLvNoSwitches.conf new file mode 100644 index 00000000..04bffe22 --- /dev/null +++ b/inputData/config/mvLvNoSwitches.conf @@ -0,0 +1,109 @@ +# Simple config to convert all models that comprise mv and lv -- either single or connected +io { + input { + csv = { + fileEncoding = "UTF-8" + fileEnding = ".csv" + separator = ";" + directoryHierarchy = false + } + } + + output { + csv = { + fileEncoding = "UTF-8" + fileEnding = ".csv" + separator = ";" + directoryHierarchy = false + } + + targetFolder = "convertedData/mvLvNoSwitches" + compress = true + } + + simbenchCodes = [ + "1-MVLV-comm-3.403-0-no_sw", + "1-MVLV-comm-3.403-1-no_sw", + "1-MVLV-comm-3.403-2-no_sw", + "1-MVLV-comm-4.416-0-no_sw", + "1-MVLV-comm-4.416-1-no_sw", + "1-MVLV-comm-4.416-2-no_sw", + "1-MVLV-comm-5.401-0-no_sw", + "1-MVLV-comm-5.401-1-no_sw", + "1-MVLV-comm-5.401-2-no_sw", + "1-MVLV-rural-1.108-0-no_sw", + "1-MVLV-rural-1.108-1-no_sw", + "1-MVLV-rural-1.108-2-no_sw", + "1-MVLV-rural-2.107-0-no_sw", + "1-MVLV-rural-2.107-1-no_sw", + "1-MVLV-rural-2.107-2-no_sw", + "1-MVLV-rural-4.101-0-no_sw", + "1-MVLV-rural-4.101-1-no_sw", + "1-MVLV-rural-4.101-2-no_sw", + "1-MVLV-semiurb-3.202-0-no_sw", + "1-MVLV-semiurb-3.202-1-no_sw", + "1-MVLV-semiurb-3.202-2-no_sw", + "1-MVLV-semiurb-4.201-0-no_sw", + "1-MVLV-semiurb-4.201-1-no_sw", + "1-MVLV-semiurb-4.201-2-no_sw", + "1-MVLV-semiurb-5.220-0-no_sw", + "1-MVLV-semiurb-5.220-1-no_sw", + "1-MVLV-semiurb-5.220-2-no_sw", + "1-MVLV-urban-5.303-0-no_sw", + "1-MVLV-urban-5.303-1-no_sw", + "1-MVLV-urban-5.303-2-no_sw", + "1-MVLV-urban-6.305-0-no_sw", + "1-MVLV-urban-6.305-1-no_sw", + "1-MVLV-urban-6.305-2-no_sw", + "1-MVLV-urban-6.309-0-no_sw", + "1-MVLV-urban-6.309-1-no_sw", + "1-MVLV-urban-6.309-2-no_sw", + "1-LV-rural1--0-no_sw", + "1-LV-rural1--1-no_sw", + "1-LV-rural1--2-no_sw", + "1-LV-rural2--0-no_sw", + "1-LV-rural2--1-no_sw", + "1-LV-rural2--2-no_sw", + "1-LV-rural3--0-no_sw", + "1-LV-rural3--1-no_sw", + "1-LV-rural3--2-no_sw", + "1-LV-semiurb4--0-no_sw", + "1-LV-semiurb4--1-no_sw", + "1-LV-semiurb4--2-no_sw", + "1-LV-semiurb5--0-no_sw", + "1-LV-semiurb5--1-no_sw", + "1-LV-semiurb5--2-no_sw", + "1-LV-urban6--0-no_sw", + "1-LV-urban6--1-no_sw", + "1-LV-urban6--2-no_sw", +// "1-MV-comm--0-no_sw", + "1-MV-comm--1-no_sw", + "1-MV-comm--2-no_sw", + "1-MV-rural--0-no_sw", + "1-MV-rural--1-no_sw", + "1-MV-rural--2-no_sw", + "1-MV-semiurb--0-no_sw", + "1-MV-semiurb--1-no_sw", + "1-MV-semiurb--2-no_sw", + "1-MV-urban--0-no_sw", + "1-MV-urban--1-no_sw", + "1-MV-urban--2-no_sw", + # Causing memory issues + #"1-MVLV-comm-all-0-no_sw", + #"1-MVLV-comm-all-1-no_sw", + #"1-MVLV-comm-all-2-no_sw", + #"1-MVLV-rural-all-0-no_sw", + #"1-MVLV-rural-all-1-no_sw", + #"1-MVLV-rural-all-2-no_sw", + #"1-MVLV-semiurb-all-0-no_sw", + #"1-MVLV-semiurb-all-1-no_sw", + #"1-MVLV-semiurb-all-2-no_sw", + #"1-MVLV-urban-all-0-no_sw", + #"1-MVLV-urban-all-1-no_sw", + #"1-MVLV-urban-all-2-no_sw", + ] +} + +conversion { + removeSwitches = true +} diff --git a/src/main/resources/config-template.conf b/src/main/resources/config-template.conf index 602a79bb..55a408d4 100644 --- a/src/main/resources/config-template.conf +++ b/src/main/resources/config-template.conf @@ -6,13 +6,20 @@ CsvConfig { directoryHierarchy = "Boolean" | false } -io.simbenchCodes = ["String"] -io.input { - download.baseUrl = "String" | "http://141.51.193.167/simbench/gui/usecase/download" - download.folder = "String" | "inputData/download/" - download.failOnExistingFiles = "Boolean" | false - csv = CsvConfig +io { + simbenchCodes = ["String"] + input { + download.baseUrl = "String" | "http://141.51.193.167/simbench/gui/usecase/download" + download.folder = "String" | "inputData/download/" + download.failOnExistingFiles = "Boolean" | false + csv = CsvConfig + } + output { + csv = CsvConfig + targetFolder = "String" | "convertedData" + compress = "Boolean" | true + } } -io.output.csv = CsvConfig -io.output.targetFolder = "String" | "convertedData" -io.output.compress = "Boolean" | true +conversion { + removeSwitches = "Boolean" | false +} \ No newline at end of file diff --git a/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala b/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala index 43744cc1..8f9caba7 100644 --- a/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala +++ b/src/main/scala/edu/ie3/simbench/config/SimbenchConfig.scala @@ -1,9 +1,10 @@ -// generated by tscfg 0.9.98 on Tue Oct 27 11:46:02 CET 2020 +// generated by tscfg 0.9.986 on Mon Aug 09 20:12:24 CEST 2021 // source: src/main/resources/config-template.conf package edu.ie3.simbench.config final case class SimbenchConfig( + conversion: SimbenchConfig.Conversion, io: SimbenchConfig.Io ) object SimbenchConfig { @@ -20,8 +21,10 @@ object SimbenchConfig { $tsCfgValidator: $TsCfgValidator ): SimbenchConfig.CsvConfig = { SimbenchConfig.CsvConfig( - directoryHierarchy = c.hasPathOrNull("directoryHierarchy") && c - .getBoolean("directoryHierarchy"), + directoryHierarchy = + c.hasPathOrNull("directoryHierarchy") && c.getBoolean( + "directoryHierarchy" + ), fileEncoding = if (c.hasPathOrNull("fileEncoding")) c.getString("fileEncoding") else "UTF-8", @@ -34,6 +37,23 @@ object SimbenchConfig { } } + final case class Conversion( + removeSwitches: scala.Boolean + ) + object Conversion { + def apply( + c: com.typesafe.config.Config, + parentPath: java.lang.String, + $tsCfgValidator: $TsCfgValidator + ): SimbenchConfig.Conversion = { + SimbenchConfig.Conversion( + removeSwitches = c.hasPathOrNull("removeSwitches") && c.getBoolean( + "removeSwitches" + ) + ) + } + } + final case class Io( input: SimbenchConfig.Io.Input, output: SimbenchConfig.Io.Output, @@ -41,7 +61,7 @@ object SimbenchConfig { ) object Io { final case class Input( - csv: CsvConfig, + csv: SimbenchConfig.CsvConfig, download: SimbenchConfig.Io.Input.Download ) object Input { @@ -60,8 +80,10 @@ object SimbenchConfig { baseUrl = if (c.hasPathOrNull("baseUrl")) c.getString("baseUrl") else "http://141.51.193.167/simbench/gui/usecase/download", - failOnExistingFiles = c.hasPathOrNull("failOnExistingFiles") && c - .getBoolean("failOnExistingFiles"), + failOnExistingFiles = + c.hasPathOrNull("failOnExistingFiles") && c.getBoolean( + "failOnExistingFiles" + ), folder = if (c.hasPathOrNull("folder")) c.getString("folder") else "inputData/download/" @@ -75,7 +97,7 @@ object SimbenchConfig { $tsCfgValidator: $TsCfgValidator ): SimbenchConfig.Io.Input = { SimbenchConfig.Io.Input( - csv = CsvConfig( + csv = SimbenchConfig.CsvConfig( if (c.hasPathOrNull("csv")) c.getConfig("csv") else com.typesafe.config.ConfigFactory.parseString("csv{}"), parentPath + "csv.", @@ -93,7 +115,7 @@ object SimbenchConfig { final case class Output( compress: scala.Boolean, - csv: CsvConfig, + csv: SimbenchConfig.CsvConfig, targetFolder: java.lang.String ) object Output { @@ -104,7 +126,7 @@ object SimbenchConfig { ): SimbenchConfig.Io.Output = { SimbenchConfig.Io.Output( compress = !c.hasPathOrNull("compress") || c.getBoolean("compress"), - csv = CsvConfig( + csv = SimbenchConfig.CsvConfig( if (c.hasPathOrNull("csv")) c.getConfig("csv") else com.typesafe.config.ConfigFactory.parseString("csv{}"), parentPath + "csv.", @@ -145,6 +167,12 @@ object SimbenchConfig { val $tsCfgValidator: $TsCfgValidator = new $TsCfgValidator() val parentPath: java.lang.String = "" val $result = SimbenchConfig( + conversion = SimbenchConfig.Conversion( + if (c.hasPathOrNull("conversion")) c.getConfig("conversion") + else com.typesafe.config.ConfigFactory.parseString("conversion{}"), + parentPath + "conversion.", + $tsCfgValidator + ), io = SimbenchConfig.Io( if (c.hasPathOrNull("io")) c.getConfig("io") else com.typesafe.config.ConfigFactory.parseString("io{}"), @@ -191,6 +219,14 @@ object SimbenchConfig { badPaths += s"'$path': ${e.getClass.getName}(${e.getMessage})" } + def addInvalidEnumValue( + path: java.lang.String, + value: java.lang.String, + enumName: java.lang.String + ): Unit = { + badPaths += s"'$path': invalid value $value for enumeration $enumName" + } + def validate(): Unit = { if (badPaths.nonEmpty) { throw new com.typesafe.config.ConfigException( diff --git a/src/main/scala/edu/ie3/simbench/convert/GridConverter.scala b/src/main/scala/edu/ie3/simbench/convert/GridConverter.scala index 4c673712..4cb5540f 100644 --- a/src/main/scala/edu/ie3/simbench/convert/GridConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/GridConverter.scala @@ -5,6 +5,7 @@ import edu.ie3.datamodel.io.source.TimeSeriesMappingSource import edu.ie3.datamodel.io.source.TimeSeriesMappingSource.MappingEntry import edu.ie3.datamodel.models.input.connector.{ LineInput, + SwitchInput, Transformer2WInput, Transformer3WInput } @@ -23,6 +24,10 @@ import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.result.NodeResult import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.value.{PValue, SValue, Value} +import edu.ie3.simbench.convert.NodeConverter.AttributeOverride.{ + JoinOverride, + SubnetOverride +} import edu.ie3.simbench.convert.types.{ LineTypeConverter, Transformer2wTypeConverter @@ -30,6 +35,7 @@ import edu.ie3.simbench.convert.types.{ import edu.ie3.simbench.exception.ConversionException import edu.ie3.simbench.model.datamodel.{ GridModel, + Line, Node, NodePFResult, Switch, @@ -38,8 +44,9 @@ import edu.ie3.simbench.model.datamodel.{ } import java.util.UUID -import scala.collection.immutable +import scala.annotation.tailrec import scala.jdk.CollectionConverters._ +import scala.collection.parallel.CollectionConverters._ case object GridConverter extends LazyLogging { @@ -47,13 +54,15 @@ case object GridConverter extends LazyLogging { * Converts a full simbench grid into power system data models [[JointGridContainer]]. Additionally, individual time * series for all system participants are delivered as well. * - * @param simbenchCode Simbench code, that is used as identifier for the grid - * @param gridInput Total grid input model to be converted + * @param simbenchCode Simbench code, that is used as identifier for the grid + * @param gridInput Total grid input model to be converted + * @param removeSwitches Whether or not to remove switches from the grid structure * @return A converted [[JointGridContainer]], a [[Vector]] of [[IndividualTimeSeries]] as well as a [[Vector]] of [[NodeResult]]s */ def convert( simbenchCode: String, - gridInput: GridModel + gridInput: GridModel, + removeSwitches: Boolean ): ( JointGridContainer, Vector[IndividualTimeSeries[_ <: PValue]], @@ -61,7 +70,8 @@ case object GridConverter extends LazyLogging { Vector[NodeResult] ) = { logger.debug(s"Converting raw grid elements of '${gridInput.simbenchCode}'") - val (rawGridElements, nodeConversion) = convertGridElements(gridInput) + val (rawGridElements, nodeConversion) = + convertGridElements(gridInput, removeSwitches) logger.debug( s"Converting system participants and their time series of '${gridInput.simbenchCode}'" @@ -94,25 +104,47 @@ case object GridConverter extends LazyLogging { /** * Converts all elements that do form the grid itself. * - * @param gridInput Total grid input model to convert + * @param gridInput Total grid input model to convert + * @param removeSwitches Whether or not to remove switches from the grid structure * @return All grid elements in converted form + a mapping from old to new node models */ def convertGridElements( - gridInput: GridModel + gridInput: GridModel, + removeSwitches: Boolean ): (RawGridElements, Map[Node, NodeInput]) = { /* Set up a sub net converter, by crawling all nodes */ val subnetConverter = SubnetConverter( gridInput.nodes.map(node => (node.vmR, node.subnet)) ) - val firstNodeConversion = convertNodes(gridInput, subnetConverter) + val slackNodeKeys = NodeConverter.getSlackNodeKeys( + gridInput.externalNets, + gridInput.powerPlants, + gridInput.res + ) - /* Update the subnet attribute at all nodes within transformer's upstream switchgear */ - val nodeConversion = updateSubnetInSwitchGears( - firstNodeConversion, - subnetConverter, - gridInput + /* Collect overriding attributes for node conversion, based on special constellations within the grid */ + val subnetOverrides = determineSubnetOverrides( + gridInput.transformers2w, + gridInput.transformers3w, + gridInput.switches, + gridInput.lines, + subnetConverter ) + val joinOverrides = if (removeSwitches) { + /* If switches are meant to be removed, join all nodes at closed switches */ + determineJoinOverrides(gridInput.switches, slackNodeKeys) + } else + Vector.empty + + val nodeConversion = + convertNodes( + gridInput.nodes, + slackNodeKeys, + subnetConverter, + subnetOverrides, + joinOverrides + ) val lines = convertLines(gridInput, nodeConversion).toSet.asJava val transformers2w = @@ -122,216 +154,122 @@ case object GridConverter extends LazyLogging { "Creation of three winding transformers is not yet implemented." ) val switches = - SwitchConverter.convert(gridInput.switches, nodeConversion).toSet.asJava + if (!removeSwitches) + SwitchConverter.convert(gridInput.switches, nodeConversion).toSet.asJava + else + Set.empty[SwitchInput].asJava val measurements = MeasurementConverter .convert(gridInput.measurements, nodeConversion) .toSet .asJava + val connectedNodes = filterIsolatedNodes( + nodeConversion, + lines, + transformers2w, + transformers3w, + switches + ) + ( new RawGridElements( - nodeConversion.values.toSet.asJava, + connectedNodes.values.toSet.asJava, lines, transformers2w, transformers3w, switches, measurements ), - nodeConversion + connectedNodes ) } /** - * Converts all apparent nodes to the equivalent power system data model. The slack information is derived from the - * attributes of external nets, power plants and renewable energy sources. + * Determine all relevant subnet override information. SimBench has a different notion of where the border of a + * subnet is. This is especially the case, if there is switch gear "upstream" of a transformer. For SimBench all + * nodes upstream of the transformer belong to the higher grid. However, for PowerSystemDataModel we expect the + * switch gear to belong to the lower grid (the transformer is in), as in a partitioned simulation, one most likely + * will control the switches in a manner, that the lower grid needs for. Therefore, for all nodes that are upstream + * of a transformer's hv node and connected via switches, explicit subnet numbers are provided. * - * @param gridInput Total grid input model to convert - * @param subnetConverter Converter holding the mapping information from simbench to power system data model sub grid - * @return A map from simbench to power system data model nodes + * @param transformers2w Collection of two winding transformers + * @param transformers3w Collection of three winding transformers + * @param switches Collection of switches + * @param lines Collection of lines + * @param subnetConverter Converter to determine subnet numbers + * @return A collection of [[SubnetOverride]]s */ - private def convertNodes( - gridInput: GridModel, + private def determineSubnetOverrides( + transformers2w: Vector[Transformer2W], + transformers3w: Vector[Transformer3W], + switches: Vector[Switch], + lines: Vector[Line[_]], subnetConverter: SubnetConverter - ): Map[Node, NodeInput] = { - val slackNodeKeys = NodeConverter.getSlackNodeKeys( - gridInput.externalNets, - gridInput.powerPlants, - gridInput.res - ) - gridInput.nodes - .map( - node => - node -> NodeConverter.convert(node, slackNodeKeys, subnetConverter) - ) - .toMap - } - - /** - * Convert the given [[NodePFResult]]s with the help of yet known conversion mapping of nodes - * - * @param input Vector of [[NodePFResult]] to convert - * @param nodeConversion Mapping from SimBench to psdm node model - * @return A [[Vector]] of converted [[NodeResult]] - */ - private def convertNodeResults( - input: Vector[NodePFResult], - nodeConversion: Map[Node, NodeInput] - ): Vector[NodeResult] = input.map { nodePfResult => - val node = nodeConversion.getOrElse( - nodePfResult.node, - throw ConversionException( - s"Cannot convert power flow result for node ${nodePfResult.node}, as the needed node conversion cannot be found." - ) - ) - NodePFResultConverter.convert(nodePfResult, node) - } - - /** - * Traverse upstream of switch chain at transformer's high voltage nodes to comply the subnet assignment to - * conventions found in PowerSystemDataModel: Those nodes will also belong to the inferior sub grid. - * - * @param initialNodeConversion Mapping from SimBench to psdm nodes, that needs update - * @param subnetConverter Converter holding information about subnet mapping - * @param gridModel Overall grid model with all needed topological models - * @return An updated mapping from SimBench to psdm nodes - */ - def updateSubnetInSwitchGears( - initialNodeConversion: Map[Node, NodeInput], - subnetConverter: SubnetConverter, - gridModel: GridModel - ): Map[Node, NodeInput] = { - val updatedConversion = - gridModel.transformers2w.foldLeft(initialNodeConversion) { - case (conversion, transformer) => - val relevantSubnet = subnetConverter.convert( - transformer.nodeLV.vmR, - transformer.nodeLV.subnet - ) - - updateAlongSwitchChain( - conversion, - transformer, - gridModel, - relevantSubnet - ) - } - - gridModel.transformers3w.foldLeft(updatedConversion) { - case (conversion, transformer) => - val relevantSubnet = subnetConverter.convert( - transformer.nodeHV.vmR, - transformer.nodeHV.subnet - ) - - updateAlongSwitchChain( - conversion, - transformer, - gridModel, - relevantSubnet - ) - } - } - - /** - * Traveling along a switch chain starting from a starting node and stopping at dead ends and those nodes, that are - * directly related to lines or transformers (read: not only switches). During this travel, every node we come along - * is updated to the relevant subnet and the mapping from SimBench to psdm is updated as well. - * - * @param initialNodeConversion Mapping from SimBench to psdm model that needs update - * @param startTransformer Transformer from whose high voltage node the travel is supposed to start - * @param gridModel Model of the grid to take information about lines and transformers from - * @param relevantSubnet The subnet id to set - * @return updated mapping from SimBench to psdm node model - */ - private def updateAlongSwitchChain( - initialNodeConversion: Map[Node, NodeInput], - startTransformer: Transformer2W, - gridModel: GridModel, - relevantSubnet: Int - ): Map[Node, NodeInput] = { - val startNode = startTransformer.nodeHV - val junctions = (gridModel.lines.flatMap( + ): Vector[SubnetOverride] = { + /* All nodes, at which a branch element is connected */ + val junctions = (lines.flatMap( line => Vector(line.nodeA, line.nodeB) - ) ++ gridModel.transformers2w - .filter(_ != startTransformer) + ) ++ transformers2w .flatMap( transformer => Vector(transformer.nodeHV, transformer.nodeLV) - ) ++ gridModel.transformers3w.flatMap( + ) ++ transformers3w.flatMap( transformer => Vector(transformer.nodeHV, transformer.nodeLV, transformer.nodeMV) )).distinct - updateAlongSwitchChain( - initialNodeConversion, - startNode, - gridModel.switches, - junctions, - relevantSubnet - ) - } - /** - * Traveling along a switch chain starting from a starting node and stopping at dead ends and those nodes, that are - * directly related to lines or transformers (read: not only switches). During this travel, every node we come along - * is updated to the relevant subnet and the mapping from SimBench to psdm is updated as well. - * - * @param initialNodeConversion Mapping from SimBench to psdm model that needs update - * @param startTransformer Transformer from whose high voltage node the travel is supposed to start - * @param gridModel Model of the grid to take information about lines and transformers from - * @param relevantSubnet The subnet id to set - * @return updated mapping from SimBench to psdm node model - */ - private def updateAlongSwitchChain( - initialNodeConversion: Map[Node, NodeInput], - startTransformer: Transformer3W, - gridModel: GridModel, - relevantSubnet: Int - ): Map[Node, NodeInput] = { - val startNode = startTransformer.nodeHV - val junctions = (gridModel.lines.flatMap( - line => Vector(line.nodeA, line.nodeB) - ) ++ gridModel.transformers2w.flatMap( - transformer => Vector(transformer.nodeHV, transformer.nodeLV) - ) ++ gridModel.transformers3w - .filter(_ != startTransformer) - .flatMap( - transformer => - Vector(transformer.nodeHV, transformer.nodeLV, transformer.nodeMV) - )).distinct - updateAlongSwitchChain( - initialNodeConversion, - startNode, - gridModel.switches, - junctions, - relevantSubnet - ) + transformers2w.flatMap { transformer => + val relevantSubnet = subnetConverter.convert( + transformer.nodeLV.vmR, + transformer.nodeLV.subnet + ) + val startNode = transformer.nodeHV + determineSubnetOverrides( + startNode, + switches, + junctions.filterNot(_ == startNode), + relevantSubnet + ) + } ++ transformers3w.flatMap { transformer => + val relevantSubnet = subnetConverter.convert( + transformer.nodeHV.vmR, + transformer.nodeHV.subnet + ) + val startNode = transformer.nodeHV + determineSubnetOverrides( + startNode, + switches, + junctions.filterNot(_ == startNode), + relevantSubnet + ) + } } /** * Traveling along a switch chain starting from a starting node and stopping at dead ends and those nodes, that are - * marked explicitly as junctions. During this travel, every node we come along is updated to the relevant subnet and - * the mapping from SimBench to psdm is updated as well. The subnet at junctions and dead ends is not altered. Adding - * the traveled nodes to the list of junctions, prevents from running in circles forever. Pay attention, that when - * starting from the hv node of a transformer, it may not be included in the set of junction nodes, if it is not part - * of any other junction. + * marked explicitly as junctions. During this travel, every node we come along gets a [[SubnetOverride]] instance + * assigned, marking, that later in node conversion, this explicit subnet number shall be used. The subnet at + * junctions and dead ends is not altered. Adding the traveled nodes to the list of junctions, prevents from running + * in circles forever. Pay attention, that when starting from the hv node of a transformer, it may not be included + * in the set of junction nodes, if it is not part of any other junction. * - * @param initialNodeConversion Mapping from SimBench to psdm model that needs update - * @param startNode Start node from which the travel is supposed to start - * @param switches Collection of all switches to consider - * @param junctions Collection of nodes that are meant to be junctions - * @param relevantSubnet The subnet id to set - * @return updated mapping from SimBench to psdm node model + * @param startNode Node, where the traversal begins + * @param switches Collection of all switches + * @param junctions Collection of nodes, that are junctions + * @param relevantSubnet The explicit subnet number to use later + * @param overrides Current collection of overrides (for recursive usage) + * @return A collection of [[SubnetOverride]]s for this traversal */ - private def updateAlongSwitchChain( - initialNodeConversion: Map[Node, NodeInput], + private def determineSubnetOverrides( startNode: Node, switches: Vector[Switch], junctions: Vector[Node], - relevantSubnet: Int - ): Map[Node, NodeInput] = { + relevantSubnet: Int, + overrides: Vector[SubnetOverride] = Vector.empty + ): Vector[SubnetOverride] = { /* If the start node is among the junctions, do not travel further (Attention: Start node should not be among * junctions when the traversing starts, otherwise nothing will happen at all.) */ if (junctions.contains(startNode)) - return initialNodeConversion + return overrides /* Get all switches, that are connected to the current starting point. If the other end of the switch is a junction, * don't follow this path, as the other side wouldn't be touched anyways. */ @@ -346,25 +284,16 @@ case object GridConverter extends LazyLogging { if (nextSwitches.isEmpty) { /* There is no further switch, therefore the end is reached -> return the new mapping. Please note, as the subnet * of the current node is only altered, if there is a next switch available, dead end nodes are not altered. */ - initialNodeConversion + overrides } else { /* Copy new node and add it to the mapping */ - val updatedNode = initialNodeConversion - .getOrElse( - startNode, - throw ConversionException( - s"Cannot update the subnet of conversion of '$startNode', as its conversion cannot be found." - ) - ) - .copy() - .subnet(relevantSubnet) - .build() - val updatedConversion = initialNodeConversion + (startNode -> updatedNode) - val newJunctions = junctions.appended(startNode) + val subnetOverride = SubnetOverride(startNode.getKey, relevantSubnet) + val updatedOverrides = overrides :+ subnetOverride + val updatedJunctions = junctions.appended(startNode) /* For all possible next upcoming switches -> Traverse along each branch (depth first search) */ - nextSwitches.foldLeft(updatedConversion) { - case (conversion, switch) => + nextSwitches.foldLeft(updatedOverrides) { + case (currentOverride, switch) => /* Determine the next node */ val nextNode = Vector(switch.nodeA, switch.nodeB).find(_ != startNode) match { @@ -375,17 +304,181 @@ case object GridConverter extends LazyLogging { ) } - return updateAlongSwitchChain( - conversion, + return determineSubnetOverrides( nextNode, switches, - newJunctions, - relevantSubnet + updatedJunctions, + relevantSubnet, + currentOverride ) } } } + /** + * Determine join overrides for all nodes, that are connected by closed switches + * + * @param switches Collection of all (closed) switches + * @param slackNodeKeys Collection of node keys, that are foreseen to be slack nodes + * @return A collection of [[JoinOverride]]s + */ + private def determineJoinOverrides( + switches: Vector[Switch], + slackNodeKeys: Vector[Node.NodeKey] + ): Vector[JoinOverride] = { + val closedSwitches = switches.filter(_.cond) + val switchGroups = determineSwitchGroups(closedSwitches) + switchGroups + .flatMap( + switchGroup => + determineJoinOverridesForSwitchGroup(switchGroup, slackNodeKeys) + ) + .distinct + } + + /** + * Determine groups of directly connected switches + * + * @param switches Collection of switches to group + * @param switchGroups Current collection of switch groups (for recursion) + * @return A collection of switch collections, that denote groups + */ + @tailrec + private def determineSwitchGroups( + switches: Vector[Switch], + switchGroups: Vector[Vector[Switch]] = Vector.empty + ): Vector[Vector[Switch]] = switches.headOption match { + case Some(currentSwitch) => + val currentNodes = Vector(currentSwitch.nodeA, currentSwitch.nodeB) + val (group, remainingSwitches) = switches.partition { switch => + currentNodes.contains(switch.nodeA) || currentNodes.contains( + switch.nodeB + ) + } + val updatedGroups = switchGroups :+ group + determineSwitchGroups(remainingSwitches, updatedGroups) + case None => + switchGroups + } + + /** + * Determine overrides per switch group + * + * @param switchGroup A group of directly connected switches + * @param slackNodeKeys Collection of node keys, that are foreseen to be slack nodes + * @return A collection of [[JoinOverride]]s + */ + private def determineJoinOverridesForSwitchGroup( + switchGroup: Vector[Switch], + slackNodeKeys: Vector[Node.NodeKey] + ): Vector[JoinOverride] = { + val nodes = + switchGroup.flatMap(switch => Vector(switch.nodeA, switch.nodeB)) + val leadingNode = + nodes.find(node => slackNodeKeys.contains(node.getKey)).getOrElse { + /* Get the node with the most occurrences */ + nodes.groupBy(identity).view.mapValues(_.size).toMap.maxBy(_._2)._1 + } + val leadingNodeKey = leadingNode.getKey + nodes.distinct + .filterNot(_ == leadingNode) + .map(node => JoinOverride(node.getKey, leadingNodeKey)) + } + + /** + * Converts all apparent nodes to the equivalent power system data model. The slack information is derived from the + * attributes of external nets, power plants and renewable energy sources. + * + * @param nodes All nodes to convert + * @param slackNodeKeys Node identifier for those, that are foreseen to be slack nodes + * @param subnetConverter Converter holding the mapping information from simbench to power system data model sub grid + * @param subnetOverrides Collection of explicit subnet assignments + * @param joinOverrides Collection of pairs of nodes, that are meant to be joined + * @return A map from simbench to power system data model nodes + */ + private def convertNodes( + nodes: Vector[Node], + slackNodeKeys: Vector[Node.NodeKey], + subnetConverter: SubnetConverter, + subnetOverrides: Vector[SubnetOverride], + joinOverrides: Vector[JoinOverride] + ): Map[Node, NodeInput] = { + val nodeToExplicitSubnet = subnetOverrides.map { + case SubnetOverride(key, subnet) => key -> subnet + }.toMap + + /* First convert all nodes, that are target of node joins */ + val nodeToJoinMap = joinOverrides.map { + case JoinOverride(key, joinWith) => key -> joinWith + }.toMap + val joinTargetNodeKeys = nodeToJoinMap.values.toSeq.distinct + val (joinTargetNodes, remainingNodes) = + nodes.partition(node => joinTargetNodeKeys.contains(node.getKey)) + val joinTargetConversion = joinTargetNodes.par + .map( + node => + node -> NodeConverter.convert( + node, + slackNodeKeys, + subnetConverter, + nodeToExplicitSubnet.get(node.getKey) + ) + ) + .seq + .toMap + + /* Then map all nodes to be joined to the converted target nodes */ + val (nodesToBeJoined, singleNodes) = remainingNodes.partition( + node => nodeToJoinMap.keySet.contains(node.getKey) + ) + val conversionWithJoinedNodes = joinTargetConversion ++ nodesToBeJoined.par + .map { node => + node -> joinTargetConversion.getOrElse( + node, + throw ConversionException( + s"The node with key '${node.getKey}' was meant to be joined with another node, but that converted target node is not apparent." + ) + ) + } + .seq + .toMap + + /* Finally convert all left over nodes */ + conversionWithJoinedNodes ++ singleNodes.par + .map( + node => + node -> NodeConverter.convert( + node, + slackNodeKeys, + subnetConverter, + nodeToExplicitSubnet.get(node.getKey) + ) + ) + .seq + .toMap + } + + /** + * Convert the given [[NodePFResult]]s with the help of yet known conversion mapping of nodes + * + * @param input Vector of [[NodePFResult]] to convert + * @param nodeConversion Mapping from SimBench to psdm node model + * @return A [[Vector]] of converted [[NodeResult]] + */ + private def convertNodeResults( + input: Vector[NodePFResult], + nodeConversion: Map[Node, NodeInput] + ): Vector[NodeResult] = + input.par.map { nodePfResult => + val node = nodeConversion.getOrElse( + nodePfResult.node, + throw ConversionException( + s"Cannot convert power flow result for node ${nodePfResult.node}, as the needed node conversion cannot be found." + ) + ) + NodePFResultConverter.convert(nodePfResult, node) + }.seq + /** * Converts the given lines. * @@ -422,6 +515,46 @@ case object GridConverter extends LazyLogging { ) } + /** + * Filter the given node conversion for only those nodes, that are also part of any branch + * + * @param nodeConversion Mapping from original node to converted model + * @param lines Collection of all lines + * @param transformers2w Collection of all two winding transformers + * @param transformers3w Collection of all three winding transformers + * @param switches Collection of all switches + * @return A mapping, that only contains connected nodes + */ + private def filterIsolatedNodes( + nodeConversion: Map[Node, NodeInput], + lines: java.util.Set[LineInput], + transformers2w: java.util.Set[Transformer2WInput], + transformers3w: java.util.Set[Transformer3WInput], + switches: java.util.Set[SwitchInput] + ): Map[Node, NodeInput] = { + val branchNodes = lines.asScala.flatMap( + line => Vector(line.getNodeA, line.getNodeB) + ) ++ transformers2w.asScala.flatMap( + transformer => Vector(transformer.getNodeA, transformer.getNodeB) + ) ++ transformers3w.asScala.flatMap( + transformer => + Vector(transformer.getNodeA, transformer.getNodeB, transformer.getNodeC) + ) ++ switches.asScala.flatMap( + switch => Vector(switch.getNodeA, switch.getNodeB) + ) + nodeConversion.partition { + case (_, convertedNode) => branchNodes.contains(convertedNode) + } match { + case (connectedNodes, unconnectedNodes) => + if (unconnectedNodes.nonEmpty) + logger.warn( + "The nodes with following keys are not part of any branch (aka. isolated) and will be neglected in the sequel.\n\t{}", + unconnectedNodes.map(_._1.getKey).mkString("\n\t") + ) + connectedNodes + } + } + /** * Converts all system participants and extracts their individual power time series * diff --git a/src/main/scala/edu/ie3/simbench/convert/LineConverter.scala b/src/main/scala/edu/ie3/simbench/convert/LineConverter.scala index fe9a7f92..c1d6e2a3 100644 --- a/src/main/scala/edu/ie3/simbench/convert/LineConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/LineConverter.scala @@ -17,6 +17,8 @@ import javax.measure.quantity.ElectricPotential import tech.units.indriya.ComparableQuantity import tech.units.indriya.quantity.Quantities +import scala.collection.parallel.CollectionConverters._ + case object LineConverter extends LazyLogging { /** @@ -32,7 +34,7 @@ case object LineConverter extends LazyLogging { types: Map[(LineType,ComparableQuantity[ElectricPotential]), LineTypeInput], nodes: Map[Node, NodeInput] ): Vector[LineInput] = - inputs.flatMap { + inputs.par.flatMap { case acLine: Line.ACLine => val (nodeA, nodeB) = NodeConverter.getNodes(acLine.nodeA, acLine.nodeB, nodes) @@ -44,7 +46,7 @@ case object LineConverter extends LazyLogging { ) Some(convert(acLine, lineType, nodeA, nodeB)) case _: Line.DCLine => None - } + }.seq /** * Converts a single [[Line]] to [[LineInput]]. [[Line.DCLine]] is not converted, as the ie3 data model does not diff --git a/src/main/scala/edu/ie3/simbench/convert/LoadConverter.scala b/src/main/scala/edu/ie3/simbench/convert/LoadConverter.scala index 3e2393c4..8aa9f6cb 100644 --- a/src/main/scala/edu/ie3/simbench/convert/LoadConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/LoadConverter.scala @@ -20,18 +20,22 @@ import edu.ie3.util.quantities.PowerSystemUnits.{ } import tech.units.indriya.quantity.Quantities +import scala.collection.parallel.CollectionConverters._ + case object LoadConverter extends ShuntConverter { def convert( loads: Vector[Load], nodes: Map[Node, NodeInput], profiles: Map[LoadProfileType, LoadProfile] - ): Map[LoadInput, IndividualTimeSeries[SValue]] = { - (for (load <- loads) yield { - val node = NodeConverter.getNode(load.node, nodes) - val profile = PowerProfileConverter.getProfile(load.profile, profiles) - convert(load, node, profile) - }).toMap - } + ): Map[LoadInput, IndividualTimeSeries[SValue]] = + loads.par + .map { load => + val node = NodeConverter.getNode(load.node, nodes) + val profile = PowerProfileConverter.getProfile(load.profile, profiles) + convert(load, node, profile) + } + .seq + .toMap /** * Converts a single SimBench [[Load]] to ie3's [[LoadInput]]. Currently not sufficiently covered: diff --git a/src/main/scala/edu/ie3/simbench/convert/NodeConverter.scala b/src/main/scala/edu/ie3/simbench/convert/NodeConverter.scala index 26b4bfbc..f39e61b6 100644 --- a/src/main/scala/edu/ie3/simbench/convert/NodeConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/NodeConverter.scala @@ -27,16 +27,18 @@ case object NodeConverter { /** * Converts a SimBench node to a PowerSystemDataModel node * - * @param input SimBench [[Node]] to convert - * @param slackNodeKeys Vector of keys, undoubtedly identifying slack nodes by (id, subnet, voltLvl) - * @param subnetConverter Subnet converter, that is initialized with the apparent SimBench subnets - * @param uuid UUID to use for the model generation (default: Random UUID) - * @return A [[NodeInput]] + * @param input SimBench [[Node]] to convert + * @param slackNodeKeys Vector of keys, undoubtedly identifying slack nodes by (id, subnet, voltLvl) + * @param subnetConverter Subnet converter, that is initialized with the apparent SimBench subnets + * @param maybeExplicitSubnet Optional explicit subnet number to assign + * @param uuid UUID to use for the model generation (default: Random UUID) + * @return A [[NodeInput]] */ def convert( input: Node, slackNodeKeys: Vector[NodeKey], subnetConverter: SubnetConverter, + maybeExplicitSubnet: Option[Int], uuid: UUID = UUID.randomUUID() ): NodeInput = { val vTarget = input.vmSetp match { @@ -48,7 +50,9 @@ case object NodeConverter { slackNodeKeys.contains(input.getKey) val geoPosition = CoordinateConverter.convert(input.coordinate) val voltLvl = VoltLvlConverter.convert(input.voltLvl, vRated) - val subnet = subnetConverter.convert(input.vmR, input.subnet) + val subnet = maybeExplicitSubnet.getOrElse( + subnetConverter.convert(input.vmR, input.subnet) + ) new NodeInput( uuid, @@ -150,4 +154,32 @@ case object NodeConverter { s"Cannot find conversion result for node ${nodeIn.id}" ) ) + + sealed trait AttributeOverride { + val key: Node.NodeKey + } + object AttributeOverride { + + /** + * Denote a subnet override for a node conversion + * + * @param key Key of the node to consider this override for + * @param targetSubnet The target subnet to use + */ + final case class SubnetOverride( + override val key: Node.NodeKey, + targetSubnet: Int + ) extends AttributeOverride + + /** + * Denote, that two nodes are foreseen to be joined + * + * @param key Key of the node to join + * @param joinWith Key of other node to join with + */ + final case class JoinOverride( + override val key: Node.NodeKey, + joinWith: Node.NodeKey + ) extends AttributeOverride + } } diff --git a/src/main/scala/edu/ie3/simbench/convert/PowerPlantConverter.scala b/src/main/scala/edu/ie3/simbench/convert/PowerPlantConverter.scala index 2ca50bd6..217050bd 100644 --- a/src/main/scala/edu/ie3/simbench/convert/PowerPlantConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/PowerPlantConverter.scala @@ -21,7 +21,7 @@ import edu.ie3.util.quantities.PowerSystemUnits.{ } import tech.units.indriya.quantity.Quantities -import scala.math._ +import scala.collection.parallel.CollectionConverters._ case object PowerPlantConverter extends ShuntConverter { @@ -37,14 +37,16 @@ case object PowerPlantConverter extends ShuntConverter { powerPlants: Vector[PowerPlant], nodes: Map[Node, NodeInput], profiles: Map[PowerPlantProfileType, PowerPlantProfile] - ): Map[FixedFeedInInput, IndividualTimeSeries[PValue]] = { - (for (powerPlant <- powerPlants) yield { - val node = NodeConverter.getNode(powerPlant.node, nodes) - val profile = - PowerProfileConverter.getProfile(powerPlant.profile, profiles) - convert(powerPlant, node, profile) - }).toMap - } + ): Map[FixedFeedInInput, IndividualTimeSeries[PValue]] = + powerPlants.par + .map { powerPlant => + val node = NodeConverter.getNode(powerPlant.node, nodes) + val profile = + PowerProfileConverter.getProfile(powerPlant.profile, profiles) + convert(powerPlant, node, profile) + } + .seq + .toMap /** * Converts a single power plant model to a fixed feed in model, as the power system data model does not reflect diff --git a/src/main/scala/edu/ie3/simbench/convert/ResConverter.scala b/src/main/scala/edu/ie3/simbench/convert/ResConverter.scala index 8b98cfea..23937022 100644 --- a/src/main/scala/edu/ie3/simbench/convert/ResConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/ResConverter.scala @@ -8,7 +8,6 @@ import edu.ie3.datamodel.models.input.system.characteristic.CosPhiFixed import edu.ie3.datamodel.models.input.{NodeInput, OperatorInput} import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.value.PValue -import edu.ie3.simbench.convert.PowerPlantConverter.cosPhi import edu.ie3.simbench.convert.profiles.PowerProfileConverter import edu.ie3.simbench.model.datamodel.profiles.{ResProfile, ResProfileType} import edu.ie3.simbench.model.datamodel.{Node, RES} @@ -19,7 +18,7 @@ import edu.ie3.util.quantities.PowerSystemUnits.{ } import tech.units.indriya.quantity.Quantities -import scala.math.{atan, cos, round} +import scala.collection.parallel.CollectionConverters._ case object ResConverter extends ShuntConverter { @@ -35,14 +34,16 @@ case object ResConverter extends ShuntConverter { res: Vector[RES], nodes: Map[Node, NodeInput], profiles: Map[ResProfileType, ResProfile] - ): Map[FixedFeedInInput, IndividualTimeSeries[PValue]] = { - (for (plant <- res) yield { - val node = NodeConverter.getNode(plant.node, nodes) - val profile = - PowerProfileConverter.getProfile(plant.profile, profiles) - convert(plant, node, profile) - }).toMap - } + ): Map[FixedFeedInInput, IndividualTimeSeries[PValue]] = + res.par + .map { plant => + val node = NodeConverter.getNode(plant.node, nodes) + val profile = + PowerProfileConverter.getProfile(plant.profile, profiles) + convert(plant, node, profile) + } + .seq + .toMap /** * Converts a single renewable energy source system to a fixed feed in model due to lacking information to diff --git a/src/main/scala/edu/ie3/simbench/convert/SubnetConverter.scala b/src/main/scala/edu/ie3/simbench/convert/SubnetConverter.scala index 4c025502..4da5f7e1 100644 --- a/src/main/scala/edu/ie3/simbench/convert/SubnetConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/SubnetConverter.scala @@ -19,7 +19,7 @@ import scala.util.matching.Regex * * Please note, that there is another difference in subnet mapping when switchgear comes into play upstream of a * transformer. As this can only be considered, when the whole grid structure is available, this is addressed in - * [[GridConverter.updateSubnetInSwitchGears()]] + * [[GridConverter.determineSubnetOverrides()]] * * @param ratedVoltageIdPairs Vector of known combinations of rated voltage and subnet id */ diff --git a/src/main/scala/edu/ie3/simbench/convert/SwitchConverter.scala b/src/main/scala/edu/ie3/simbench/convert/SwitchConverter.scala index 2531cf74..733517c7 100644 --- a/src/main/scala/edu/ie3/simbench/convert/SwitchConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/SwitchConverter.scala @@ -7,6 +7,8 @@ import edu.ie3.datamodel.models.input.connector.SwitchInput import edu.ie3.datamodel.models.input.{NodeInput, OperatorInput} import edu.ie3.simbench.model.datamodel.{Node, Switch} +import scala.collection.parallel.CollectionConverters._ + case object SwitchConverter { /** @@ -20,11 +22,11 @@ case object SwitchConverter { switches: Vector[Switch], nodes: Map[Node, NodeInput] ): Vector[SwitchInput] = - for (input <- switches) yield { + switches.par.map { input => val (nodeA, nodeB) = NodeConverter.getNodes(input.nodeA, input.nodeB, nodes) convert(input, nodeA, nodeB) - } + }.seq /** * Converts a [[Switch]] into ie3's [[SwitchInput]] diff --git a/src/main/scala/edu/ie3/simbench/convert/Transformer2wConverter.scala b/src/main/scala/edu/ie3/simbench/convert/Transformer2wConverter.scala index 149fb8c8..4a36c43d 100644 --- a/src/main/scala/edu/ie3/simbench/convert/Transformer2wConverter.scala +++ b/src/main/scala/edu/ie3/simbench/convert/Transformer2wConverter.scala @@ -10,6 +10,8 @@ import edu.ie3.simbench.exception.ConversionException import edu.ie3.simbench.model.datamodel.types.Transformer2WType import edu.ie3.simbench.model.datamodel.{Node, Transformer2W} +import scala.collection.parallel.CollectionConverters._ + case object Transformer2wConverter { /** @@ -25,7 +27,7 @@ case object Transformer2wConverter { types: Map[Transformer2WType, Transformer2WTypeInput], nodes: Map[Node, NodeInput] ): Vector[Transformer2WInput] = - for (input <- inputs) yield { + inputs.par.map { input => val (nodeA, nodeB) = NodeConverter.getNodes(input.nodeHV, input.nodeLV, nodes) val transformerType = types.getOrElse( @@ -36,7 +38,7 @@ case object Transformer2wConverter { ) convert(input, transformerType, nodeA, nodeB) - } + }.seq /** * Converts a SimBench [[Transformer2W]] into a [[Transformer2WInput]]. diff --git a/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala b/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala index 1c1900fd..4482ad04 100644 --- a/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala +++ b/src/main/scala/edu/ie3/simbench/main/RunSimbench.scala @@ -15,7 +15,12 @@ import edu.ie3.simbench.model.SimbenchCode import edu.ie3.util.io.FileIOUtils import org.apache.commons.io.FilenameUtils +import scala.concurrent.Await +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration.Duration import scala.jdk.CollectionConverters._ +import scala.jdk.FutureConverters.CompletionStageOps +import scala.util.{Failure, Success} /** * This is not meant to be final production code. It is more a place for "testing" the full method stack. @@ -72,7 +77,11 @@ object RunSimbench extends SimbenchHelper { timeSeriesMapping, powerFlowResults ) = - GridConverter.convert(simbenchCode, simbenchModel) + GridConverter.convert( + simbenchCode, + simbenchModel, + simbenchConfig.conversion.removeSwitches + ) logger.info(s"$simbenchCode - Writing converted data set to files") /* Check, if a directory hierarchy is needed or not */ @@ -110,9 +119,19 @@ object RunSimbench extends SimbenchHelper { val archivePath = Paths.get( FilenameUtils.concat(baseTargetDirectory, simbenchCode + ".tar.gz") ) - FileIOUtils.compressDir(rawOutputPath, archivePath) + val compressFuture = + FileIOUtils.compressDir(rawOutputPath, archivePath).asScala + compressFuture.onComplete { + case Success(_) => + FileIOUtils.deleteRecursively(rawOutputPath) + case Failure(exception) => + logger.error( + s"Compression of output files to '$archivePath' has failed. Keep raw data.", + exception + ) + } - FileIOUtils.deleteRecursively(rawOutputPath) + Await.ready(compressFuture, Duration("180s")) } } } diff --git a/src/test/scala/edu/ie3/simbench/convert/GridConverterSpec.scala b/src/test/scala/edu/ie3/simbench/convert/GridConverterSpec.scala index fed781ea..b4714e1a 100644 --- a/src/test/scala/edu/ie3/simbench/convert/GridConverterSpec.scala +++ b/src/test/scala/edu/ie3/simbench/convert/GridConverterSpec.scala @@ -2,64 +2,190 @@ package edu.ie3.simbench.convert import java.nio.file.Paths import java.util - import edu.ie3.datamodel.models.UniqueEntity import edu.ie3.datamodel.models.input.NodeInput import edu.ie3.datamodel.models.input.connector.{LineInput, Transformer2WInput} import edu.ie3.datamodel.models.input.system.{FixedFeedInInput, LoadInput} +import edu.ie3.simbench.convert.NodeConverter.AttributeOverride.JoinOverride import edu.ie3.simbench.io.SimbenchReader -import edu.ie3.simbench.model.datamodel.GridModel -import edu.ie3.test.common.UnitSpec +import edu.ie3.simbench.model.datamodel.{GridModel, Node, Switch} +import edu.ie3.test.common.{SwitchTestingData, UnitSpec} import org.scalatest.Inside._ import scala.jdk.CollectionConverters._ -class GridConverterSpec extends UnitSpec { +class GridConverterSpec extends UnitSpec with SwitchTestingData { val simbenchReader: SimbenchReader = SimbenchReader( "1-LV-rural1--0-no_sw", Paths.get("src/test/resources/gridData/1-LV-rural1--0-no_sw") ) val input: GridModel = simbenchReader.readGrid() - "The grid converter" should { - "bring the correct amount of converted models" in { - val actual = GridConverter.convert("1-LV-rural1--0-no_sw", input) - inside(actual) { - case (gridContainer, timeSeries, timeSeriesMapping, powerFlowResults) => - /* Evaluate the correctness of the container by counting the occurrence of models (the correct conversion is - * tested in separate unit tests */ - gridContainer.getGridName shouldBe "1-LV-rural1--0-no_sw" - countClassOccurrences(gridContainer.getRawGrid.allEntitiesAsList()) shouldBe Map( - classOf[NodeInput] -> 15, - classOf[LineInput] -> 13, - classOf[Transformer2WInput] -> 1 - ) - countClassOccurrences( - gridContainer.getSystemParticipants.allEntitiesAsList() - ) shouldBe Map( - classOf[FixedFeedInInput] -> 4, - classOf[LoadInput] -> 13 - ) - countClassOccurrences(gridContainer.getGraphics.allEntitiesAsList()) shouldBe Map - .empty[Class[_ <: UniqueEntity], Int] - - /* Evaluate the correctness of the time series by counting the occurrence of models */ - timeSeries.size shouldBe 17 - - /* Evaluate the existence of time series mappings for all participants */ - timeSeriesMapping.size shouldBe 17 - val participantUuids = gridContainer.getSystemParticipants - .allEntitiesAsList() - .asScala - .map(_.getUuid) - .toVector - /* There is no participant uuid in mapping, that is not among participants */ - timeSeriesMapping.exists( - entry => !participantUuids.contains(entry.getParticipant) - ) shouldBe false - - /* Evaluate the amount of converted power flow results */ - powerFlowResults.size shouldBe 15 + "The grid converter" when { + "joining nodes around closed switches" should { + val determineJoinOverridesForSwitchGroup = + PrivateMethod[Vector[JoinOverride]]( + Symbol("determineJoinOverridesForSwitchGroup") + ) + "determine join overrides in a switch group of only one switch correctly" in { + val actual = GridConverter invokePrivate determineJoinOverridesForSwitchGroup( + switchGroup2, + Vector.empty[Node.NodeKey] + ) + actual.size shouldBe 1 + val nodeKeys = actual.flatMap { + case JoinOverride(key, joinWith) => Vector(key, joinWith) + } + nodeKeys.contains(nodeH.getKey) shouldBe true + nodeKeys.contains(nodeI.getKey) shouldBe true + } + + "determine join overrides in a switch group of two switches correctly" in { + val actual = GridConverter invokePrivate determineJoinOverridesForSwitchGroup( + switchGroup0, + Vector.empty[Node.NodeKey] + ) + actual.size shouldBe 2 + + val targets = actual.map(_.joinWith).distinct + targets.size shouldBe 1 + targets.head shouldBe nodeB.getKey + } + + "determine join overrides in a switch group of three switches with central node correctly" in { + val actual = GridConverter invokePrivate determineJoinOverridesForSwitchGroup( + switchGroup1, + Vector.empty[Node.NodeKey] + ) + actual.size shouldBe 3 + + val targets = actual.map(_.joinWith).distinct + targets.size shouldBe 1 + targets.head shouldBe nodeD.getKey + } + + "determine join overrides in a switch group of two switches correctly, if slack node keys are given" in { + val actual = GridConverter invokePrivate determineJoinOverridesForSwitchGroup( + switchGroup0, + Vector(nodeC.getKey) + ) + actual.size shouldBe 2 + + val targets = actual.map(_.joinWith).distinct + targets.size shouldBe 1 + targets.head shouldBe nodeC.getKey + } + + val determineSwitchGroups = + PrivateMethod[Vector[Vector[Switch]]](Symbol("determineSwitchGroups")) + "determine switch groups correctly" in { + val actual = GridConverter invokePrivate determineSwitchGroups( + switches, + Vector.empty[Vector[Switch]] + ) + + /* We expect three groups */ + actual.size shouldBe 3 + /* We expect all switches to be covered */ + actual.flatten.distinct.size shouldBe 6 + + actual.sortBy(_.size).foreach { group => + if (group.contains(switchAB)) { + group.contains(switchBC) shouldBe true + } else if (group.contains(switchDE)) { + group.contains(switchDF) shouldBe true + group.contains(switchDG) shouldBe true + } else { + group.contains(switchHI) shouldBe true + } + } + } + + val determineJoinOverrides = + PrivateMethod[Vector[JoinOverride]](Symbol("determineJoinOverrides")) + "determine join overrides for the whole grid correctly" in { + val actual = GridConverter invokePrivate determineJoinOverrides( + switches, + Vector(nodeC.getKey) + ) + + val joinMap = actual.map { + case JoinOverride(key, joinWith) => key -> joinWith + }.toMap + + joinMap.size shouldBe 6 + joinMap.values.toSeq.distinct.size shouldBe 3 + + val expected = Map( + nodeA.getKey -> nodeC.getKey, + nodeB.getKey -> nodeC.getKey, + nodeE.getKey -> nodeD.getKey, + nodeF.getKey -> nodeD.getKey, + nodeG.getKey -> nodeD.getKey, + nodeH.getKey -> nodeI.getKey, + nodeI.getKey -> nodeH.getKey + ) + + joinMap.foreach { + case key -> joinWith => + expected.get(key) match { + case Some(expectedJoin) => + joinWith shouldBe expectedJoin + case None => fail(s"No mapping expected for node key '$key'") + } + } + } + } + + "converting a full data set" should { + "bring the correct amount of converted models" in { + val actual = GridConverter.convert( + "1-LV-rural1--0-no_sw", + input, + removeSwitches = false + ) + inside(actual) { + case ( + gridContainer, + timeSeries, + timeSeriesMapping, + powerFlowResults + ) => + /* Evaluate the correctness of the container by counting the occurrence of models (the correct conversion is + * tested in separate unit tests */ + gridContainer.getGridName shouldBe "1-LV-rural1--0-no_sw" + countClassOccurrences(gridContainer.getRawGrid.allEntitiesAsList()) shouldBe Map( + classOf[NodeInput] -> 15, + classOf[LineInput] -> 13, + classOf[Transformer2WInput] -> 1 + ) + countClassOccurrences( + gridContainer.getSystemParticipants.allEntitiesAsList() + ) shouldBe Map( + classOf[FixedFeedInInput] -> 4, + classOf[LoadInput] -> 13 + ) + countClassOccurrences(gridContainer.getGraphics.allEntitiesAsList()) shouldBe Map + .empty[Class[_ <: UniqueEntity], Int] + + /* Evaluate the correctness of the time series by counting the occurrence of models */ + timeSeries.size shouldBe 17 + + /* Evaluate the existence of time series mappings for all participants */ + timeSeriesMapping.size shouldBe 17 + val participantUuids = gridContainer.getSystemParticipants + .allEntitiesAsList() + .asScala + .map(_.getUuid) + .toVector + /* There is no participant uuid in mapping, that is not among participants */ + timeSeriesMapping.exists( + entry => !participantUuids.contains(entry.getParticipant) + ) shouldBe false + + /* Evaluate the amount of converted power flow results */ + powerFlowResults.size shouldBe 15 + } } } } diff --git a/src/test/scala/edu/ie3/simbench/convert/NodeConverterSpec.scala b/src/test/scala/edu/ie3/simbench/convert/NodeConverterSpec.scala index 08116489..7fef1dae 100644 --- a/src/test/scala/edu/ie3/simbench/convert/NodeConverterSpec.scala +++ b/src/test/scala/edu/ie3/simbench/convert/NodeConverterSpec.scala @@ -120,7 +120,13 @@ class NodeConverterSpec extends UnitSpec with ConverterTestData { "The node converter" should { "build a slack node with all possible information given correctly" in { val actual = - NodeConverter.convert(slackNode, slackNodeKeys, subnetConverter, uuid) + NodeConverter.convert( + slackNode, + slackNodeKeys, + subnetConverter, + None, + uuid + ) actual.getId shouldBe slackNodeExpected.getId actual.getOperationTime shouldBe slackNodeExpected.getOperationTime @@ -133,7 +139,13 @@ class NodeConverterSpec extends UnitSpec with ConverterTestData { "build a normal node with least possible information given correctly" in { val actual = - NodeConverter.convert(otherNode, slackNodeKeys, subnetConverter, uuid) + NodeConverter.convert( + otherNode, + slackNodeKeys, + subnetConverter, + None, + uuid + ) actual.getId shouldBe otherNodeExpected.getId actual.getOperationTime shouldBe otherNodeExpected.getOperationTime actual.getOperator shouldBe otherNodeExpected.getOperator diff --git a/src/test/scala/edu/ie3/test/common/ConfigTestData.scala b/src/test/scala/edu/ie3/test/common/ConfigTestData.scala index cacd3b0d..ae9f0162 100644 --- a/src/test/scala/edu/ie3/test/common/ConfigTestData.scala +++ b/src/test/scala/edu/ie3/test/common/ConfigTestData.scala @@ -2,6 +2,7 @@ package edu.ie3.test.common import edu.ie3.simbench.config import edu.ie3.simbench.config.SimbenchConfig +import edu.ie3.simbench.config.SimbenchConfig.Conversion import edu.ie3.simbench.config.SimbenchConfig.Io.Input trait ConfigTestData { @@ -31,5 +32,7 @@ trait ConfigTestData { List("1-LV-urban6--0-sw", "blabla", "1-EHVHV-mixed-2-0-sw") ) - val validConfig = new SimbenchConfig(validIo) + val validConversionConfig: Conversion = Conversion(removeSwitches = false) + + val validConfig = new SimbenchConfig(validConversionConfig, validIo) } diff --git a/src/test/scala/edu/ie3/test/common/SwitchTestingData.scala b/src/test/scala/edu/ie3/test/common/SwitchTestingData.scala new file mode 100644 index 00000000..4e109205 --- /dev/null +++ b/src/test/scala/edu/ie3/test/common/SwitchTestingData.scala @@ -0,0 +1,160 @@ +package edu.ie3.test.common + +import edu.ie3.simbench.model.datamodel.{Node, Switch} +import edu.ie3.simbench.model.datamodel.enums.{NodeType, SwitchType} + +trait SwitchTestingData { + val nodeA: Node = Node( + "nodeA", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeB: Node = Node( + "nodeB", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeC: Node = Node( + "nodeC", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeD: Node = Node( + "nodeD", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeE: Node = Node( + "nodeE", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeF: Node = Node( + "nodeF", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeG: Node = Node( + "nodeG", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeH: Node = Node( + "nodeH", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + val nodeI: Node = Node( + "nodeI", + NodeType.BusBar, + vmR = BigDecimal("0.4"), + vmMin = BigDecimal("0.9"), + vmMax = BigDecimal("1.1"), + subnet = "a", + voltLvl = 7 + ) + + /* First switch chain between nodes A, B and C */ + val switchAB: Switch = Switch( + "switchAB", + nodeA, + nodeB, + SwitchType.LoadSwitch, + cond = true, + subnet = "a", + voltLvl = 7, + substation = None + ) + val switchBC: Switch = Switch( + "switchBC", + nodeB, + nodeC, + SwitchType.LoadSwitch, + cond = true, + subnet = "a", + voltLvl = 7, + substation = None + ) + val switchGroup0 = Vector(switchAB, switchBC) + + /* Second switch chain between nodes D, E, F und G */ + val switchDE: Switch = Switch( + "switchDE", + nodeD, + nodeE, + SwitchType.LoadSwitch, + cond = true, + subnet = "a", + voltLvl = 7, + substation = None + ) + val switchDF: Switch = Switch( + "switchDF", + nodeD, + nodeF, + SwitchType.LoadSwitch, + cond = true, + subnet = "a", + voltLvl = 7, + substation = None + ) + val switchDG: Switch = Switch( + "switchDG", + nodeD, + nodeG, + SwitchType.LoadSwitch, + cond = true, + subnet = "a", + voltLvl = 7, + substation = None + ) + val switchGroup1 = Vector(switchDE, switchDF, switchDG) + + /* Third switch "chain" between node H and I */ + val switchHI: Switch = Switch( + "switchHI", + nodeH, + nodeI, + SwitchType.LoadSwitch, + cond = true, + subnet = "a", + voltLvl = 7, + substation = None + ) + val switchGroup2 = Vector(switchHI) + + val switches = + Vector(switchAB, switchBC, switchDE, switchDF, switchDG, switchHI) +}