Merge branch 'dev' into sp/#24-kafka-result-output

.github/dependabot.yml

@@ -12,11 +12,22 @@ updates:
   - johanneshiry
   - t-ober
   - sensarmad
+  - sebastian-peter
+  - danielfeismann
   ignore:
     - dependency-name: org.spockframework:spock-core
       versions:
-        - 2.1-groovy-3.0-SNAPSHOT
         - 2.1-groovy-2.5-SNAPSHOT
+        - 2.1-groovy-3.0-SNAPSHOT
+        - 2.2-groovy-4.0-SNAPSHOT
+        - 2.2-groovy-2.5-SNAPSHOT
+        - 2.2-groovy-3.0-SNAPSHOT
+        - 2.2-M1-groovy-2.5
+        - 2.2-M1-groovy-3.0
+        - 2.2-M1-groovy-4.0
+        - 2.2-M2-groovy-2.5
+        - 2.2-M2-groovy-3.0
+        - 2.2-M2-groovy-4.0
     - dependency-name: org.scalatest:scalatest_2.13
       versions:
         - 3.3.0-SNAP+

.gitignore

@@ -225,9 +225,10 @@ output
 deploy.properties
 
 # pv model it data
-!src/test/resources/**/it-data
+!src/test/resources/**/it
 
 # avoid ignoring test jars
 !src/test/resources/**/*.jar
+
 # Ignore Gradle build output directory
 build

CHANGELOG.md

@@ -5,10 +5,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Added
+- Implement SQL source for primary data [#34](https://github.com/ie3-institute/simona/issues/34)
+
 ### Changed
 - Improving code readability in EvcsAgent by moving FreeLotsRequest to separate methods
+- Re-organizing test resources into their respective packages [#105](https://github.com/ie3-institute/simona/issues/105)
+- BREAKING: Using snapshot version of PSDM
+- Simplified PrimaryServiceProxy due to changes in PSDM [#120](https://github.com/ie3-institute/simona/issues/120)
 
 ### Fixed
 - Location of `vn_simona` test grid (was partially in Berlin and Dortmund)
+- Let `ParticipantAgent` die after failed registration with secondary services (prevents stuck simulation)
 
 [Unreleased]: https://github.com/ie3-institute/simona/compare/a14a093239f58fca9b2b974712686b33e5e5f939...HEAD

build.gradle

@@ -7,10 +7,10 @@ plugins {
   id 'signing'
   id 'maven-publish' // publish to a maven repo (local or mvn central, has to be defined)
   id 'pmd' // code check, working on source code
-  id 'com.diffplug.spotless' version '6.2.0'// code format
+  id 'com.diffplug.spotless' version '6.3.0'// code format
   id 'com.github.onslip.gradle-one-jar' version '1.0.6' // pack a self contained jar
-  id "com.github.ben-manes.versions" version '0.41.0'
-  id "de.undercouch.download" version "4.1.2" // downloads plugin
+  id "com.github.ben-manes.versions" version '0.42.0'
+  id "de.undercouch.download" version "5.0.2" // downloads plugin
   id "kr.motd.sphinx" version "2.10.1" // documentation generation
   id "com.github.johnrengelman.shadow" version "7.1.2" // fat jar
   id "org.sonarqube" version "3.3" // sonarqube
@@ -25,9 +25,12 @@ ext {
   javaVersion = JavaVersion.VERSION_17
 
   scalaVersion = '2.13'
-  scalaBinaryVersion = '2.13.7'
+  scalaBinaryVersion = '2.13.8'
   akkaVersion = '2.6.18'
   tscfgVersion = '0.9.997'
+  scapegoatVersion = '1.4.12'
+
+  testContainerVersion = '0.40.3'
 
   scriptsLocation = 'gradle' + File.separator + 'scripts' + File.separator // location of script plugins
 }
@@ -61,14 +64,20 @@ repositories {
 }
 
 dependencies {
+  constraints {
+    implementation( 'junit:junit:4.13.2+' ){
+      because "CVE-2020-15250 - Temporary folder vulnerability - https://github.com/advisories/GHSA-269g-pwp5-87pp"
+    }
+  }
+
   // ie³ internal repository
   implementation('com.github.ie3-institute:PowerSystemUtils:1.6') {
     exclude group: 'org.apache.logging.log4j'
     exclude group: 'org.slf4j'
     /* Exclude our own nested dependencies */
     exclude group: 'com.github.ie3-institute'
   }
-  implementation('com.github.ie3-institute:PowerSystemDataModel:2.1.0') {
+  implementation('com.github.ie3-institute:PowerSystemDataModel:3.0-SNAPSHOT') {
     exclude group: 'org.apache.logging.log4j'
     exclude group: 'org.slf4j'
     /* Exclude our own nested dependencies */
@@ -90,17 +99,21 @@ dependencies {
 
   /* logging */
   implementation "com.typesafe.scala-logging:scala-logging_${scalaVersion}:3.9.4" // akka scala logging
-  implementation "ch.qos.logback:logback-classic:1.2.10"
+  implementation "ch.qos.logback:logback-classic:1.2.11"
 
   /* testing */
-  testImplementation 'org.spockframework:spock-core:2.1-M2-groovy-3.0'
+  testImplementation 'org.spockframework:spock-core:2.1-groovy-3.0'
   testImplementation 'org.scalatestplus:mockito-3-4_2.13:3.2.10.0'
-  implementation 'org.mockito:mockito-core:4.2.0' // mocking framework
-  testImplementation "org.scalatest:scalatest_${scalaVersion}:3.2.10"
-  testRuntimeClasspath 'com.vladsch.flexmark:flexmark-all:0.62.2'
+  implementation 'org.mockito:mockito-core:4.4.0' // mocking framework
+  testImplementation "org.scalatest:scalatest_${scalaVersion}:3.2.11"
+  testRuntimeClasspath 'com.vladsch.flexmark:flexmark-all:0.64.0'
   testImplementation group: 'org.pegdown', name: 'pegdown', version: '1.6.0'
   testImplementation "com.typesafe.akka:akka-testkit_${scalaVersion}:${akkaVersion}" // akka testkit
 
+  // testcontainers
+  testImplementation "com.dimafeng:testcontainers-scala-scalatest_${scalaVersion}:${testContainerVersion}"
+  testImplementation "com.dimafeng:testcontainers-scala-postgresql_${scalaVersion}:${testContainerVersion}"
+
   /* --- Scala libs --- */
   /* CORE Scala */
   implementation "org.scala-lang:scala-library:${scalaBinaryVersion}"
@@ -115,7 +128,7 @@ dependencies {
   implementation "com.typesafe.akka:akka-cluster-tools_${scalaVersion}:${akkaVersion}"
 
   /* config */
-  implementation 'com.typesafe:config:1.4.1'
+  implementation 'com.typesafe:config:1.4.2'
   implementation "com.github.carueda:tscfg_2.13:$tscfgVersion"
   implementation "com.github.scopt:scopt_${scalaVersion}:4.0.1" // cmd args parser
 
@@ -126,8 +139,8 @@ dependencies {
   implementation 'org.locationtech.jts.io:jts-io-common:1.18.2'
 
   /* Scala compiler plugin for static code analysis */
-  implementation "com.sksamuel.scapegoat:scalac-scapegoat-plugin_${scalaBinaryVersion}:1.4.12"
-  scalaCompilerPlugin "com.sksamuel.scapegoat:scalac-scapegoat-plugin_${scalaBinaryVersion}:1.4.11"
+  implementation "com.sksamuel.scapegoat:scalac-scapegoat-plugin_${scalaBinaryVersion}:${scapegoatVersion}"
+  scalaCompilerPlugin "com.sksamuel.scapegoat:scalac-scapegoat-plugin_${scalaBinaryVersion}:${scapegoatVersion}"
 
   /* Kafka */
   implementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.0.0'
@@ -136,14 +149,14 @@ dependencies {
   testImplementation 'org.testcontainers:kafka:1.16.2' // kafka testing
 
   implementation 'org.apache.commons:commons-math3:3.6.1' // apache commons math3
-  implementation 'org.apache.poi:poi-ooxml:5.2.0' // used for FilenameUtils
+  implementation 'org.apache.poi:poi-ooxml:5.2.1' // used for FilenameUtils
   implementation 'javax.measure:unit-api:2.1.3'
-  implementation 'tech.units:indriya:2.1.2' // quantities
+  implementation 'tech.units:indriya:2.1.3' // quantities
   implementation 'org.apache.commons:commons-csv:1.9.0'
   implementation 'org.scalanlp:breeze_2.13:1.3' // scientific calculations (http://www.scalanlp.org/)
   implementation 'de.lmu.ifi.dbs.elki:elki:0.7.5' // Statistics (for random load model)
-  implementation 'com.google.guava:guava:31.0.1-jre' // Building threads
-  implementation 'org.jgrapht:jgrapht-core:1.4.0'
+  implementation 'com.google.guava:guava:31.1-jre' // Building threads
+  implementation 'org.jgrapht:jgrapht-core:1.5.1'
 }
 
 tasks.withType(JavaCompile) {

docs/readthedocs/usersguide.rst

@@ -17,11 +17,11 @@ To run and customize the project you need a Java Development Kit (JDK) installat
 Installation
 ============
 
-You can find and download the source code of the latest stable SIMONA version `here <https://git.ie3.e-technik.tu-dortmund.de/SIMONACrew/SIMONA>`_. Go ahead and clone the repository using git:
+You can find and download the source code of the latest stable SIMONA version `here <https://github.com/ie3-institute/simona>`_. Go ahead and clone the repository using git:
 
 .. code-block:: none
 
-   $ git clone https://git.ie3.e-technik.tu-dortmund.de/SIMONACrew/SIMONA.git
+   $ git clone https://github.com/ie3-institute/simona.git
 
 
 Running a Standalone Simulation
@@ -170,7 +170,7 @@ SIMONA is capable of running an external sub-simulation by integration within th
 The information flow between SIMONA and the external simulation is partitioned into a control stream (see ``edu.ie3.simona.api.ExtSimAdapter``) and a number of optional data streams.
 Currently, only a data stream transporting electric vehicle movement information is implemented (see ``edu.ie3.simona.service.ev.ExtEvDataService``).
 
-An external simulation has to depend on `SimonaAPI <https://git.ie3.e-technik.tu-dortmund.de/SIMONACrew/SimonaAPI>`_ and make use of some of its interfaces (see below).
+An external simulation has to depend on `SimonaAPI <https://github.com/ie3-institute/simonaAPI>`_ and make use of some of its interfaces (see below).
 In order to run an external simulation, several requirements have to be fulfilled and a bunch of preparation steps have to be followed.
 
 .. note::
@@ -200,3 +200,25 @@ These steps have to be performed each time updates to the external simulation ne
 - Copy the resulting *jar* (usually placed inside <external project>/build/libs) to SIMONA/inputData/ext_sim.
 
 Now, when a simulation with SIMONA is started (see `above <#running-a-standalone-simulation>`_), the external simulation is triggered at each tick that it requested.
+
+Troubleshooting
+===============
+
+My power flow calculation isn't converging - why is that?
+---------------------------------------------------------
+
+When your power flow is not converging it means that the load situation in the grid during the time of the power flow calculation is not physically feasible.
+
+This can have basically one of the following two reasons:
+
+#.
+    There is more load in the grid than it can physically handle.
+
+#.
+    There is more generation in the grid than it can physically handle.
+
+One of the main reasons is a misconfiguration of the grid and its assets.
+Assess the power of the load and generation units and check if the values make sense.
+Keep in mind the metric prefixes that are assumed for the models, which are listed in the `PSDM docs <https://powersystemdatamodel.readthedocs.io/en/latest/index.html>`_.
+If everything seems to be configured correctly it could also be the case that the grid itself is incorrectly configured.
+Do a similar sanity check for the grids assets.

gradle/scripts/scoverage.gradle

@@ -3,7 +3,7 @@
 // https://github.com/scoverage/gradle-scoverage/issues/109 for details
 
 scoverage {
-  scoverageVersion = "1.4.10"
+  scoverageVersion = "1.4.11"
   scoverageScalaVersion = scalaBinaryVersion
   coverageOutputHTML = false
   coverageOutputXML = true

gradle/scripts/tscfg.gradle

@@ -15,7 +15,7 @@ task genConfigClass {
       args = [
         "build/tscfg-${tscfgVersion}.jar",
         "--spec",
-        "src/main/resources/config/simona-config-template.conf",
+        "src/main/resources/config/config-template.conf",
         "--scala",
         "--durations",
         "--pn",

input/samples/vn_simona/fullGrid/pv_input.csv

@@ -1,4 +1,4 @@
-"uuid","albedo","azimuth","cos_phi_rated","eta_conv","height","id","k_g","k_t","market_reaction","node","operates_from","operates_until","operator","q_characteristics","s_rated"
+"uuid","albedo","azimuth","cos_phi_rated","eta_conv","elevation_angle","id","k_g","k_t","market_reaction","node","operates_from","operates_until","operator","q_characteristics","s_rated"
 5b38af42-1ee4-4a41-b666-ea141187df37,0.20000000298023224,-11.463644027709961,0.8999999761581421,96.0,33.62879943847656,NS_NET146_F2_(3)_PV,0.8999999761581421,1.0,false,0170837a-1876-45f9-a613-666f9991964d,,,,cosPhiFixed:{(0.00,0.90)},10.0
 e447506e-3d43-4bce-8aab-a7ca8b7fbc45,0.20000000298023224,3.8914573192596436,0.8999999761581421,98.0,42.77021408081055,NS_NET146_F4_(9)_PV,0.8999999761581421,1.0,false,9b889b73-c108-4b38-b6eb-3377841e0c83,,,,cosPhiFixed:{(0.00,0.90)},10.0
 6cac0624-6336-4418-bcf0-990abcdb824b,0.20000000298023224,-8.097375869750977,0.8999999761581421,98.0,44.90728759765625,NS_NET146_F4_(16)_PV,0.8999999761581421,1.0,false,9f7599de-c488-46c5-b053-1279a511f7b9,,,,cosPhiFixed:{(0.00,0.90)},30.0

src/main/resources/config/config-template.conf

@@ -101,9 +101,7 @@ simona.input.primary = {
       jdbcUrl:  string
       userName: string
       password: string
-      weatherTableName: string
       schemaName: string | "public"
-      timeColumnName: string
       timePattern: string | "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'" # default pattern from PSDM:TimeBasedSimpleValueFactory
     }
     #@optional
@@ -150,9 +148,9 @@ simona.input.weather.datasource = {
       jdbcUrl:  string
       userName: string
       password: string
-      weatherTableName: string
+      tableName: string
       schemaName: string | "public"
-      timeColumnName: string
+      timePattern: string | "yyyy-MM-dd'T'HH:mm:ss[.S[S][S]]'Z'" # default pattern from PSDM:TimeBasedSimpleValueFactory
     }
     #@optional
     couchbaseParams = {

src/main/scala/edu/ie3/simona/agent/grid/DBFSAlgorithm.scala

@@ -52,14 +52,17 @@ import java.time.{Duration, ZonedDateTime}
 import java.util.UUID
 import javax.measure.Quantity
 import javax.measure.quantity.{Dimensionless, ElectricPotential}
-import scala.concurrent.Future
+import scala.concurrent.{ExecutionContext, Future}
 
 /** Trait that is normally mixed into every [[GridAgent]] to enable distributed
   * forward backward sweep (DBFS) algorithm execution. It is considered to be
   * the standard behaviour of a [[GridAgent]].
   */
 trait DBFSAlgorithm extends PowerFlowSupport with GridResultsSupport {
   this: GridAgent =>
+  // implicit ExecutionContext should be in scope
+  // see https://doc.akka.io/docs/akka/2.5/futures.html
+  implicit val ec: ExecutionContext = context.dispatcher
 
   when(SimulateGrid) {
 
@@ -1191,16 +1194,16 @@ trait DBFSAlgorithm extends PowerFlowSupport with GridResultsSupport {
       gridAgentBaseData: GridAgentBaseData,
       currentTimestamp: ZonedDateTime
   ): Unit = {
-    // otherwise .last will throw an exception
-    if (gridAgentBaseData.sweepValueStores.nonEmpty) {
-      notifyListener(
-        this.createResultModels(
-          gridAgentBaseData.gridEnv.gridModel,
-          gridAgentBaseData.sweepValueStores.last._2
-        )(
-          currentTimestamp
+    gridAgentBaseData.sweepValueStores.lastOption.foreach {
+      case (_, valueStore) =>
+        notifyListener(
+          this.createResultModels(
+            gridAgentBaseData.gridEnv.gridModel,
+            valueStore
+          )(
+            currentTimestamp
+          )
         )
-      )
     }
   }