From 497617e318852855636ebb36e86b8ba162a3f060 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 17 Feb 2023 12:36:40 +0100 Subject: [PATCH 1/8] Adding grid validation. --- .../ie3/simona/io/grid/CsvGridSource.scala | 80 ------------------- .../edu/ie3/simona/io/grid/GridProvider.scala | 32 ++++---- .../simona/model/participant/PvModelIT.groovy | 9 +-- 3 files changed, 22 insertions(+), 99 deletions(-) delete mode 100644 src/main/scala/edu/ie3/simona/io/grid/CsvGridSource.scala diff --git a/src/main/scala/edu/ie3/simona/io/grid/CsvGridSource.scala b/src/main/scala/edu/ie3/simona/io/grid/CsvGridSource.scala deleted file mode 100644 index 5bb4bdb7ee..0000000000 --- a/src/main/scala/edu/ie3/simona/io/grid/CsvGridSource.scala +++ /dev/null @@ -1,80 +0,0 @@ -/* - * © 2020. TU Dortmund University, - * Institute of Energy Systems, Energy Efficiency and Energy Economics, - * Research group Distribution grid planning and operation - */ - -package edu.ie3.simona.io.grid - -import edu.ie3.datamodel.io.naming.FileNamingStrategy -import edu.ie3.datamodel.io.source.csv._ -import edu.ie3.datamodel.models.input.container._ - -import scala.jdk.OptionConverters._ - -object CsvGridSource { - - def readGrid( - gridName: String, - csvSep: String, - baseFolder: String, - fileNamingStrategy: FileNamingStrategy - ): Option[JointGridContainer] = { - - // build the sources - val csvTypeSource: CsvTypeSource = - new CsvTypeSource(csvSep, baseFolder, fileNamingStrategy) - val csvRawGridSource: CsvRawGridSource = new CsvRawGridSource( - csvSep, - baseFolder, - fileNamingStrategy, - csvTypeSource - ) - val csvThermalSource: CsvThermalSource = new CsvThermalSource( - csvSep, - baseFolder, - fileNamingStrategy, - csvTypeSource - ) - val csvSystemParticipantSource: CsvSystemParticipantSource = - new CsvSystemParticipantSource( - csvSep, - baseFolder, - fileNamingStrategy, - csvTypeSource, - csvThermalSource, - csvRawGridSource - ) - val csvGraphicSource: CsvGraphicSource = new CsvGraphicSource( - csvSep, - baseFolder, - fileNamingStrategy, - csvTypeSource, - csvRawGridSource - ) - - // read and get the models - val rawGridElements = csvRawGridSource.getGridData.toScala - val systemParticipants = - csvSystemParticipantSource.getSystemParticipants.toScala - val graphicElements = csvGraphicSource.getGraphicElements.toScala - - (rawGridElements, systemParticipants, graphicElements) match { - case ( - Some(rawGridElements), - Some(systemParticipants), - Some(graphicElements) - ) => - Some( - new JointGridContainer( - gridName, - rawGridElements, - systemParticipants, - graphicElements - ) - ) - case (_, _, _) => None - } - } - -} diff --git a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala index b710b50a7a..ed5e248f5e 100644 --- a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala +++ b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala @@ -6,10 +6,11 @@ package edu.ie3.simona.io.grid -import edu.ie3.datamodel.io.naming.FileNamingStrategy +import com.typesafe.scalalogging.LazyLogging +import edu.ie3.datamodel.io.source.csv.CsvJointGridContainerSource import edu.ie3.datamodel.models.input.container.JointGridContainer +import edu.ie3.datamodel.utils.validation.ValidationUtils import edu.ie3.simona.config.SimonaConfig -import edu.ie3.simona.exceptions.InitializationException /** Takes [[edu.ie3.simona.config.SimonaConfig.Simona.Input.Grid.Datasource]] as * input and provides a [[JointGridContainer]] based on the configuration incl. @@ -18,7 +19,7 @@ import edu.ie3.simona.exceptions.InitializationException * @version 0.1 * @since 28.04.20 */ -object GridProvider { +object GridProvider extends LazyLogging { def gridFromConfig( simulationName: String, @@ -29,18 +30,21 @@ object GridProvider { case GridSourceType.CSV => gridDataSource.csvParams match { case Some(params) => - CsvGridSource - .readGrid( - simulationName, - params.csvSep, - params.directoryPath, - new FileNamingStrategy() - ) - .getOrElse( - throw new InitializationException( - "Error while initializing CsvGridSource! Cannot proceed without a valid GridSource!" + val jointGridContainer = CsvJointGridContainerSource.read( + simulationName, + params.csvSep, + params.directoryPath + ) + try { + ValidationUtils.check(jointGridContainer) + } catch { + case exception: Exception => + logger.warn( + s"Error while initializing CsvGridSource! Cannot proceed without a valid GridSource!", + exception ) - ) + } + jointGridContainer case None => throw new RuntimeException( "CSVGridSource requires csv params to be set!" diff --git a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy index a80c88b307..921140d931 100644 --- a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy +++ b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy @@ -6,9 +6,9 @@ package edu.ie3.simona.model.participant -import edu.ie3.datamodel.io.naming.FileNamingStrategy +import edu.ie3.datamodel.io.source.csv.CsvJointGridContainerSource +import edu.ie3.datamodel.models.input.container.JointGridContainer import edu.ie3.datamodel.models.input.system.PvInput -import edu.ie3.simona.io.grid.CsvGridSource import edu.ie3.simona.model.SystemComponent import edu.ie3.simona.model.participant.control.QControl import edu.ie3.simona.ontology.messages.services.WeatherMessage @@ -127,9 +127,8 @@ trait PvModelITHelper { HashMap createPvModels() { "load the grid input data from the corresponding resources folder" - def csvGridSource = CsvGridSource.readGrid("it_grid", ";", - this.getClass().getResource("_pv/it/grid_data").file, - new FileNamingStrategy()) + def csvGridSource = CsvJointGridContainerSource.read("it_grid", ";", + this.getClass().getResource("_pv/it/grid_data").file) def simulationStartDate = TimeUtil.withDefaults.toZonedDateTime("2011-01-01 00:00:00") def simulationEndDate = TimeUtil.withDefaults.toZonedDateTime("2012-01-01 00:00:00") From 45a4a025a6c6922854256c6a28bb1a91ec1774e3 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 17 Feb 2023 12:57:36 +0100 Subject: [PATCH 2/8] Adding changes to CHANGELOG. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4885ed4fef..01ca47f7e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added listeners to DBFS tests to check the result output and check the handling of failed power flows [#269](https://github.com/ie3-institute/simona/issues/269) - Added DBFS test with participant load and added testing for FinishGridSimulationTrigger [#281](https://github.com/ie3-institute/simona/issues/281) - Added Dependabot updates to sphinx/readthedocs dependencies [#448](https://github.com/ie3-institute/simona/issues/448) +- Check for grid validity with PSDM `ValidationUtils` [#460](https://github.com/ie3-institute/simona/issues/460) ### Changed - Re-organizing test resources into their respective packages [#105](https://github.com/ie3-institute/simona/issues/105) From b83b64c39b849747f3af55cf0d3bfe31c05c3000 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 17 Feb 2023 13:13:06 +0100 Subject: [PATCH 3/8] Fixing codacy. --- src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala index ed5e248f5e..e3d9c24cff 100644 --- a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala +++ b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala @@ -12,6 +12,8 @@ import edu.ie3.datamodel.models.input.container.JointGridContainer import edu.ie3.datamodel.utils.validation.ValidationUtils import edu.ie3.simona.config.SimonaConfig +import scala.util.{Failure, Try} + /** Takes [[edu.ie3.simona.config.SimonaConfig.Simona.Input.Grid.Datasource]] as * input and provides a [[JointGridContainer]] based on the configuration incl. * necessary sanity checks @@ -35,10 +37,9 @@ object GridProvider extends LazyLogging { params.csvSep, params.directoryPath ) - try { - ValidationUtils.check(jointGridContainer) - } catch { - case exception: Exception => + + Try.apply(ValidationUtils.check(jointGridContainer)) match { + case Failure(exception) => logger.warn( s"Error while initializing CsvGridSource! Cannot proceed without a valid GridSource!", exception From 26c58b0da57b195adb98899d832cbd3883755b6a Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Fri, 17 Feb 2023 15:12:24 +0100 Subject: [PATCH 4/8] Improving validation. --- src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala index e3d9c24cff..11c53b4a8b 100644 --- a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala +++ b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala @@ -12,7 +12,7 @@ import edu.ie3.datamodel.models.input.container.JointGridContainer import edu.ie3.datamodel.utils.validation.ValidationUtils import edu.ie3.simona.config.SimonaConfig -import scala.util.{Failure, Try} +import scala.util.{Failure, Success, Try} /** Takes [[edu.ie3.simona.config.SimonaConfig.Simona.Input.Grid.Datasource]] as * input and provides a [[JointGridContainer]] based on the configuration incl. @@ -38,12 +38,13 @@ object GridProvider extends LazyLogging { params.directoryPath ) - Try.apply(ValidationUtils.check(jointGridContainer)) match { + Try(ValidationUtils.check(jointGridContainer)) match { case Failure(exception) => logger.warn( - s"Error while initializing CsvGridSource! Cannot proceed without a valid GridSource!", - exception + s"Error while validating JointGridContainer! Cannot proceed without a valid GridSource!\n\t{}", + exception.getMessage ) + case Success(_) => logger.debug("Validation successful.") } jointGridContainer case None => From 04e9616e5e4f19ee927ed2297a941b14c1c4f243 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 20 Feb 2023 12:53:42 +0100 Subject: [PATCH 5/8] Resolving codacy issues. --- .../groovy/edu/ie3/simona/model/participant/PvModelIT.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy index 921140d931..888951751f 100644 --- a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy +++ b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy @@ -7,7 +7,6 @@ package edu.ie3.simona.model.participant import edu.ie3.datamodel.io.source.csv.CsvJointGridContainerSource -import edu.ie3.datamodel.models.input.container.JointGridContainer import edu.ie3.datamodel.models.input.system.PvInput import edu.ie3.simona.model.SystemComponent import edu.ie3.simona.model.participant.control.QControl @@ -134,7 +133,7 @@ trait PvModelITHelper { def simulationEndDate = TimeUtil.withDefaults.toZonedDateTime("2012-01-01 00:00:00") HashMap pvModels = new HashMap<>() - for (PvInput inputModel : csvGridSource.get().getSystemParticipants().getPvPlants()) { + for (PvInput inputModel : csvGridSource.getSystemParticipants().getPvPlants()) { PvModel model = PvModel.apply( inputModel.getUuid(), inputModel.getId(), @@ -173,6 +172,7 @@ trait PvModelITHelper { HashMap modelToWeatherMap if (weatherMap.containsKey(time)) { modelToWeatherMap = weatherMap.get(time) + weatherMap.put(time, modelToWeatherMap) } if (modelToWeatherMap == null) { From a14b48cecc69bc199d2c6ab2a5a226ef8a79cc01 Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Mon, 20 Feb 2023 14:03:39 +0100 Subject: [PATCH 6/8] Fixing codacy issues. --- .../simona/model/participant/PvModelIT.groovy | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy index 888951751f..a2b6cdc478 100644 --- a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy +++ b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy @@ -133,25 +133,25 @@ trait PvModelITHelper { def simulationEndDate = TimeUtil.withDefaults.toZonedDateTime("2012-01-01 00:00:00") HashMap pvModels = new HashMap<>() - for (PvInput inputModel : csvGridSource.getSystemParticipants().getPvPlants()) { + for (PvInput inputModel : csvGridSource.systemParticipants.pvPlants) { PvModel model = PvModel.apply( - inputModel.getUuid(), - inputModel.getId(), + inputModel.uuid, + inputModel.id, SystemComponent.determineOperationInterval( simulationStartDate, simulationEndDate, - inputModel.getOperationTime() + inputModel.operationTime ), 1d, - QControl.apply(inputModel.getqCharacteristics()), - inputModel.getsRated(), - inputModel.getCosPhiRated(), - inputModel.getNode().getGeoPosition().getY(), - inputModel.getNode().getGeoPosition().getX(), - inputModel.getAlbedo(), - inputModel.getEtaConv(), - inputModel.getAzimuth(), - inputModel.getElevationAngle(), + QControl.apply(inputModel.qCharacteristics), + inputModel.sRated, + inputModel.cosPhiRated, + inputModel.node.geoPosition.y, + inputModel.node.geoPosition.x, + inputModel.albedo, + inputModel.etaConv, + inputModel.azimuth, + inputModel.elevationAngle, getQuantity(1d, SQUARE_METRE) ) From 5f2d3d2655bb8b23b3114ccae78c90ae2df196da Mon Sep 17 00:00:00 2001 From: staudtMarius Date: Tue, 21 Feb 2023 13:20:18 +0100 Subject: [PATCH 7/8] Adding requested changes. --- src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala | 5 ++++- .../groovy/edu/ie3/simona/model/participant/PvModelIT.groovy | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala index 11c53b4a8b..31f4a9b6fd 100644 --- a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala +++ b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala @@ -44,7 +44,10 @@ object GridProvider extends LazyLogging { s"Error while validating JointGridContainer! Cannot proceed without a valid GridSource!\n\t{}", exception.getMessage ) - case Success(_) => logger.debug("Validation successful.") + case Success(_) => + logger.debug( + s"Validation of given grid was successful. The grid ${jointGridContainer.getGridName} is a valid grid." + ) } jointGridContainer case None => diff --git a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy index a2b6cdc478..32450e289f 100644 --- a/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy +++ b/src/test/groovy/edu/ie3/simona/model/participant/PvModelIT.groovy @@ -172,7 +172,6 @@ trait PvModelITHelper { HashMap modelToWeatherMap if (weatherMap.containsKey(time)) { modelToWeatherMap = weatherMap.get(time) - weatherMap.put(time, modelToWeatherMap) } if (modelToWeatherMap == null) { From 547b69adbd41a531e0ab4463db31cfb1c2df3fa6 Mon Sep 17 00:00:00 2001 From: Sebastian Peter Date: Tue, 21 Feb 2023 15:27:53 +0100 Subject: [PATCH 8/8] Slightly improving log messages --- src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala index 31f4a9b6fd..76b1170e4c 100644 --- a/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala +++ b/src/main/scala/edu/ie3/simona/io/grid/GridProvider.scala @@ -41,12 +41,12 @@ object GridProvider extends LazyLogging { Try(ValidationUtils.check(jointGridContainer)) match { case Failure(exception) => logger.warn( - s"Error while validating JointGridContainer! Cannot proceed without a valid GridSource!\n\t{}", + s"Validation of grid ${jointGridContainer.getGridName} failed: \n\t{}", exception.getMessage ) case Success(_) => logger.debug( - s"Validation of given grid was successful. The grid ${jointGridContainer.getGridName} is a valid grid." + s"Validation of given grid ${jointGridContainer.getGridName} was successful." ) } jointGridContainer