Optimize workflow execution (#14)

* Add tdrules cache * Optimize IT runtime * Final review
giis-uniovi · Jul 27, 2023 · 276d50b · 276d50b
1 parent 88e64d2
commit 276d50b
Show file tree

Hide file tree

Showing 26 changed files with 549 additions and 86 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -9,42 +9,29 @@ updates:
       - dependency-name: "org.slf4j*"
         update-types: ["version-update:semver-major"]
     schedule:
-      interval: weekly
-      day: "friday"
-    assignees:
-      - "javiertuya" 
+      interval: daily
     open-pull-requests-limit: 20
 
   - package-ecosystem: nuget
     directory: "/net"
-    #avoids updates that cause not compilable code
     ignore:
       - dependency-name: "Microsoft.Data.SQLite"
       - dependency-name: "Microsoft.EntityFramework*"
         update-types: ["version-update:semver-major"]
     schedule:
-      interval: weekly
-      day: "friday"
-    assignees:
-      - "javiertuya" 
+      interval: daily
     open-pull-requests-limit: 20
 
   - package-ecosystem: nuget
     directory: "/net/QACover" #reports
     ignore:
       - dependency-name: "Microsoft.Data.SQLite"
     schedule:
-      interval: weekly
-      day: "friday"
-    assignees:
-      - "javiertuya" 
+      interval: daily
     open-pull-requests-limit: 20
 
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
-      interval: weekly
-      day: "friday"
+      interval: daily
     open-pull-requests-limit: 20
-    assignees:
-      - "javiertuya" 
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -24,7 +24,7 @@ jobs:
     #if: ${{ false }}  # disable for now
     strategy:
       matrix:
-        scope: [UT, IT, Postgres, Sqlserver, Oracle]
+        scope: [UT, Postgres, Sqlserver, Oracle]
       fail-fast: false
     permissions:
       checks: write
@@ -72,6 +72,13 @@ jobs:
             gvenzl/oracle-xe:11.2.0.2-slim-faststart
           chmod u+x setup/wait-container-ready.sh && ./setup/wait-container-ready.sh test-oracle "DATABASE IS READY TO USE!"
 
+      - name: Rules cache
+        id: rules-cache
+        uses: actions/cache@v3.3.1
+        with:
+          path: qacover-core/.tdrules-cache
+          key: tdrules-cache-${{ matrix.scope }}
+
       # Run the tests UT, IT and those that require database server
 
       - name: Test unit and aggregate surefire report - ${{ matrix.scope }}
@@ -90,24 +97,6 @@ jobs:
           -Duser.timezone=Europe/Madrid
         # NOTE: must specify a timezone to avoid oracle error ORA-01882: timezone region not found
 
-      # IT requires first running the tests outside of maven, then a test case compares results
-      # runs with surefire also
-      - name: Integration test execution - ${{ matrix.scope }}
-        if: ${{ matrix.scope == 'IT' }}
-        run: |
-          cd it
-          ant install
-          ant spring-petclinic
-          ant qacover-api
-          ant install-uber
-          ant qacover-uber 
-      - name: Integration test results comparison - ${{ matrix.scope }}
-        if: ${{ matrix.scope == 'IT' }}
-        run: >
-          mvn test surefire-report:report -Daggregate=true -Dsurefire.failIfNoSpecifiedTests=false
-          -Dtest=IT*
-          -Dmaven.test.failure.ignore=true -U --no-transfer-progress
-
       # Reporting
 
       - name: Additional aggregated junit report
@@ -145,17 +134,66 @@ jobs:
             */target/failsafe-reports
             */target/*.html
             */target/*.log
-      - name: Publish additional test report files from IT artifacts
-        if: always() && matrix.scope == 'IT'
+
+  test-IT:
+    runs-on: ubuntu-latest
+    #if: ${{ false }}  # disable for now
+    permissions:
+      checks: write
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+
+      # IT requires first running the tests outside of maven, then a test case compares results
+      # runs with surefire also
+      - name: Integration test execution - IT
+        run: |
+          cd it
+          ant test-parallel
+      - name: Integration test results comparison - ${{ matrix.scope }}
+        run: >
+          mvn test surefire-report:report -Daggregate=true -Dsurefire.failIfNoSpecifiedTests=false
+          -Dtest=IT*
+          -Dmaven.test.failure.ignore=true -U --no-transfer-progress
+          
+      - name: Additional aggregated junit report
+        if: always()
+        uses: javiertuya/junit-report-action@v1.2.0
+        with:
+          surefire-files: "*/target/surefire-reports/TEST-*.xml"
+          report-dir: target/site
+          report-title: "Test Report: IT - Branch: ${{ github.ref_name }}"
+
+      - name: Generate report checks
+        if: always()
+        uses: mikepenz/action-junit-report@v3.8.0
+        with:
+          check_name: "test-result-IT"
+          report_paths: "*/target/surefire-reports/TEST-*.xml"
+          fail_on_failure: 'true'
+
+      - name: Publish test report files
+        if: always()
         uses: actions/upload-artifact@v3
         with:
-          name: "test-report-files-IT-artifacts"
+          name: "test-report-files-IT"
           path: |
-            it/*/target/surefire-reports
-            it/*/target/qacover
+            target/site
+            it/index.html
+            **/target/qacover
+            */target/surefire-reports
+            */target/failsafe-reports
+            */target/*.html
+            */target/*.log
 
   test-net:
     runs-on: ubuntu-latest
+    #if: ${{ false }}  # disable for now
     permissions:
       checks: write
     defaults:
@@ -180,6 +218,12 @@ jobs:
           # SQLServer does not have an on startup script, run it now
           docker exec test-sqlserver bash -c "chmod u+x setup.d/sqlserver-setup.sh && ./setup.d/sqlserver-setup.sh"     
       
+      - name: Rules cache
+        id: rules-cache
+        uses: actions/cache@v3.3.1
+        with:
+          path: net/.tdrules-cache
+          key: tdrules-cache-Net
       - name: Run test - ADO.NET
         run: dotnet test --logger "trx;LogFileName=../../reports/qacover-report.trx" QACoverTest/QACoverTest.csproj
       - name: Run test - Entity Framework
@@ -243,7 +287,7 @@ jobs:
           dotnet build
       
   test-report:
-    needs: [test-java, test-net]
+    needs: [test-java, test-IT, test-net]
     #if: ${{ false }}  # disable for now
     #avoid publishing to Github pages PRs and dependabot branches
     if: ${{ always() && github.event_name != 'pull_request' && !contains('/head/refs/dependabot/', github.ref) }}

diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 target
+.tdrules-cache
 pom.xml.versionsBackup
 TESTS-TestSuites.xml
 dependency-reduced-pom.xml

diff --git a/it/build.xml b/it/build.xml
@@ -1,4 +1,4 @@
-<project default="test-all"  basedir="." xmlns:if="ant:if" xmlns:unless="ant:unless" >
+<project default="test-sequential"  basedir="." xmlns:if="ant:if" xmlns:unless="ant:unless" >
 <description>Execution of integration tests, outside of the parent pom context</description>
 
 	<condition property="isWindows" value="true">
@@ -12,18 +12,41 @@
 	<xmlproperty file="../pom.xml" />
 	<echo>Integration tests, parent pom version: ${project.version}</echo>
 
+	<!-- Entry point for individual steps -->
+	<target name="install" ><install/></target>
+	<target name="spring-petclinic" ><spring-petclinic/></target>
+	<target name="qacover-api" ><qacover-api/></target>
+	<target name="install-uber" ><install-uber/></target>
+	<target name="qacover-uber" ><qacover-uber/></target>
+
 	<!-- Entry point for everything, the first install may be not needed, but ensures this script is standalone -->
-	<target name="test-all" depends="install,spring-petclinic,qacover-api,install-uber,qacover-uber" >
+	<target name="test-sequential" depends="install,spring-petclinic,qacover-api,install-uber,qacover-uber" >
+	</target>
+
+	<!-- Entry point for everything with parallel execution -->
+	<target name="test-parallel" >
+		<install/>
+		<parallel>
+			<spring-petclinic/>
+			<sequential>
+				<qacover-api/>
+				<install-uber/>
+				<qacover-uber/>
+			</sequential>
+		</parallel>
 	</target>
 
-	<target name="install">
+	<macrodef name="install">
+	  <sequential>
 		<echo>*******************************</echo>
 		<echo>*** INSTALLING DEPENDENCIES ***</echo>
 		<echo>*******************************</echo>
 		<mvn folder=".." arg1="install" arg2="-DskipTests=true" />
-	</target>
+	  </sequential>
+	</macrodef>
 
-	<target name="spring-petclinic">
+	<macrodef name="spring-petclinic">
+	  <sequential>
 		<!--
 		Full execution of a spring project that includes qacover-core as dependency.
 		Ensures that artifacts are the right version, executes tests and generate reports from commandline
@@ -37,9 +60,12 @@
 		<delete dir="spring-petclinic-main/target" />
 		<mvn folder="spring-petclinic-main" arg1="clean" arg2="test" />
 		<report rules="spring-petclinic-main/target/qacover/rules" reports="spring-petclinic-main/target/qacover/reports" />
-	</target>
+		<echo>*** END RUNNING SPRING-PETCLINIC-MAIN ***</echo>
+	  </sequential>
+	</macrodef>
 
-	<target name="qacover-api" >
+	<macrodef name="qacover-api" >
+	  <sequential>
 		<echo>******************************</echo>
 		<echo>*** RUNNING QACOVER AS API ***</echo>
 		<echo>******************************</echo>
@@ -51,16 +77,23 @@
 		<mvn folder="qacover-api-sample" arg1="versions:set" arg2="-DnewVersion=${project.version}" />
 		<delete dir="qacover-api-sample/target" />
 		<mvn folder="qacover-api-sample" arg1="clean" arg2="test" />
-	</target>	
+		<echo>*** END RUNNING QACOVER AS API ***</echo>
+	  </sequential>
+	</macrodef>	
 
-	<target name="install-uber">
+	<macrodef name="install-uber">
+	  <sequential>
 		<echo>***************************</echo>
 		<echo>*** INSTALLING UBER JAR ***</echo>
 		<echo>***************************</echo>
-		<mvn folder=".." arg2="install" arg3="-DskipTests=true" arg4="-P" arg5="uber" arg6="-pl" arg7="qacover-core" />
-	</target>
+		<!-- Only packages the uber jar to target folder -->
+		<mvn folder=".." arg2="package" arg3="-DskipTests=true" arg4="-P" arg5="uber" arg6="-pl" arg7="qacover-core" />
+		<echo>*** END INSTALLING UBER JAR ***</echo>
+	  </sequential>
+	</macrodef>
 
-	<target name="qacover-uber">
+	<macrodef name="qacover-uber">
+	  <sequential>
 		<!--
 		Execution of a java standalone jar that does not include qacover-core as dependency,
 		but adds the qacover uber jar to its classpath.
@@ -91,7 +124,9 @@
 			</classpath>
 		</java>
 		<report rules="qacover-uber-main/target/qacover/rules" reports="qacover-uber-main/target/qacover/reports" />
-	</target>
+		<echo>*** END RUNNING QACOVER-UBER-MAIN ***</echo>
+	  </sequential>
+	</macrodef>
 
 	<macrodef name="mvn" xmlns:unless="ant:unless" >
 	    <attribute name="folder" description="folder where the pom.xml is located" />
@@ -108,7 +143,7 @@
 				<arg value="${maven}" />
 				<arg value="-DskipITs=true" />
 				<arg value="-Dmaven.javadoc.skip=true" />
-				<arg value="--no-transfer-progress" />
+				<!--<arg value="- -no-transfer-progress" />-->
 				<arg value="@{arg1}" unless:blank="@{arg1}" />
 				<arg value="@{arg2}" unless:blank="@{arg2}" />
 				<arg value="@{arg3}" unless:blank="@{arg3}" />

diff --git a/it/spring-petclinic-main/pom.xml b/it/spring-petclinic-main/pom.xml
@@ -5,7 +5,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.springframework.samples</groupId>
   <artifactId>spring-petclinic</artifactId>
-  <version>1.5.0</version>
+  <version>1.5.1-SNAPSHOT</version>
 
   <parent>
     <groupId>org.springframework.boot</groupId>

diff --git a/net/QACover/Giis.Tdrules.Model.IO/ModelJsonSerializer.N.cs b/net/QACover/Giis.Tdrules.Model.IO/ModelJsonSerializer.N.cs
@@ -0,0 +1,27 @@
+using Newtonsoft.Json;
+using System;
+
+namespace Giis.Tdrules.Model.IO
+{
+    // Temporal location, to be moved to tdrules-model
+    public class ModelJsonSerializer
+    {
+        /// <summary>Recommended serialization for models (exclude null and empty attributes)</summary>
+        public string Serialize(object model, bool prettyPrint)
+        {
+            JsonSerializerSettings settings = new JsonSerializerSettings
+            {
+                NullValueHandling = NullValueHandling.Ignore
+            };
+            JsonSerializer serializer = new JsonSerializer();
+            string result = JsonConvert.SerializeObject(model, prettyPrint ? Formatting.Indented : Formatting.None, settings);
+            return result;
+        }
+
+        /// <summary>Recommended deserialization for models</summary>
+        public object Deserialize(string json, Type clazz)
+        {
+            return JsonConvert.DeserializeObject(json, clazz);
+        }
+    }
+}
diff --git a/net/QACover/Translated/Giis.Qacover.Core.Services/Configuration.cs b/net/QACover/Translated/Giis.Qacover.Core.Services/Configuration.cs
@@ -29,6 +29,8 @@ public class Configuration
 
 		private string storeRulesLocation;
 
+		private string cacheRulesLocation;
+
 		private string storeReportsLocation;
 
 		private string fpcServiceUrl;
@@ -240,6 +242,18 @@ public virtual void SetStoreRulesLocation(string location)
 			storeRulesLocation = location;
 		}
 
+		public virtual string GetCacheRulesLocation()
+		{
+			return cacheRulesLocation;
+		}
+
+		public virtual Giis.Qacover.Core.Services.Configuration SetCacheRulesLocation(string location)
+		{
+			// Ensures that location is relative to project root and not null
+			cacheRulesLocation = location != null && !string.Empty.Equals(location) ? FileUtil.GetPath(Parameters.GetProjectRoot(), location) : string.Empty;
+			return this;
+		}
+
 		public virtual string GetStoreReportsLocation()
 		{
 			return storeReportsLocation;