Test cache

.github/workflows/test.yml

@@ -72,6 +72,13 @@ jobs:
             gvenzl/oracle-xe:11.2.0.2-slim-faststart
           chmod u+x setup/wait-container-ready.sh && ./setup/wait-container-ready.sh test-oracle "DATABASE IS READY TO USE!"
 
+      - name: Cache Primes
+        id: cache-primes
+        uses: actions/cache@v3
+        with:
+          path: qacover-core/.tdrules-cache
+          key: tdrules-cache-${{ matrix.scope }}
+
       # Run the tests UT, IT and those that require database server
 
       - name: Test unit and aggregate surefire report - ${{ matrix.scope }}
@@ -156,6 +163,7 @@ jobs:
 
   test-net:
     runs-on: ubuntu-latest
+    if: ${{ false }}  # disable for now
     permissions:
       checks: write
     defaults:
@@ -244,9 +252,9 @@ jobs:
       
   test-report:
     needs: [test-java, test-net]
-    #if: ${{ false }}  # disable for now
+    if: ${{ false }}  # disable for now
     #avoid publishing to Github pages PRs and dependabot branches
-    if: ${{ always() && github.event_name != 'pull_request' && !contains('/head/refs/dependabot/', github.ref) }}
+    #if: ${{ always() && github.event_name != 'pull_request' && !contains('/head/refs/dependabot/', github.ref) }}
     runs-on: ubuntu-latest
     # Configuration to deploy at github pages
     permissions:
@@ -338,10 +346,10 @@ jobs:
 
   sonarqube:
     needs: [test-java]
-    #if: ${{ false }}  # disable for now
+    if: ${{ false }}  # disable for now
     #This job fails when comming from a dependabot PR (can't read the sonarqube token for security reasons).
     #Links to discussions and workaround at: https://github.com/giis-uniovi/samples-giis-template/issues/4
-    if: ${{ github.actor != 'dependabot[bot]' }}
+    #if: ${{ github.actor != 'dependabot[bot]' }}
     runs-on: ubuntu-latest
     steps:
       - uses: javiertuya/sonarqube-action@v1.1.0

.gitignore

@@ -1,4 +1,5 @@
 target
+.tdrules-cache
 pom.xml.versionsBackup
 TESTS-TestSuites.xml
 dependency-reduced-pom.xml

qacover-core/qacover.properties

@@ -48,7 +48,7 @@ qacover.query.infer.parameters=false
 
 # Alternative endpoint of the SqlRules service
 # Default: empty
-qacover.fpc.url=
+qacover.fpc.url=http://localhost:8080
 
 # Additional options to control the rule generation (separated by spaces)
 # Default: empty

qacover-core/src/main/java/giis/qacover/core/services/Configuration.java

@@ -26,6 +26,7 @@ public class Configuration { // NOSONAR singleton allowed
 	private static Configuration instance;
 	private String name=QACOVER_NAME;
 	private String storeRulesLocation;
+	private String cacheRulesLocation;
 	private String storeReportsLocation;
 	private String fpcServiceUrl;
 	private String fpcServiceOptions;
@@ -190,6 +191,13 @@ public String getStoreRulesLocation() {
 	public void setStoreRulesLocation(String location) {
 		storeRulesLocation = location;
 	}
+	public String getCacheRulesLocation() {
+		return cacheRulesLocation;
+	}
+	public Configuration setCacheRulesLocation(String location) {
+		cacheRulesLocation = location;
+		return this;
+	}
 	public String getStoreReportsLocation() {
 		return storeReportsLocation;
 	}

qacover-core/src/main/java/giis/qacover/core/services/RuleServices.java

@@ -12,7 +12,6 @@
 import giis.qacover.model.QueryWithParameters;
 import giis.qacover.model.SchemaModel;
 import giis.qacover.portable.QaCoverException;
-import giis.tdrules.client.TdRulesApi;
 import giis.tdrules.client.rdb.DbSchemaApi;
 import giis.tdrules.model.io.SqlRulesXmlSerializer;
 import giis.tdrules.openapi.model.DbSchema;
@@ -66,7 +65,7 @@ public String getRulesInput(String sql, SchemaModel schemaModel, String fpcOptio
 
 	public String[] getAllTableNames(String sql) {
 		this.setErrorContext("Get query table names");
-		SqlTableListBody model = getApi().sqlTablesPost(sql);
+		SqlTableListBody model = getApi().getTables(sql);
 		injectFaultIfNeeded(model);
 		if (!"".equals(model.getError()))
 			throw new QaCoverException(model.getError());
@@ -75,7 +74,7 @@ public String[] getAllTableNames(String sql) {
 
 	public QueryWithParameters inferQueryWithParameters(String sql) {
 		this.setErrorContext("Infer query parameters");
-		SqlParametersBody model = getApi().sqlParametersPost(sql);
+		SqlParametersBody model = getApi().getParameters(sql);
 		injectFaultIfNeeded(model);
 		if (!"".equals(model.getError()))
 			throw new QaCoverException(model.getError());

qacover-core/src/main/java/giis/qacover/core/services/TdRulesApi.java

@@ -0,0 +1,71 @@
+package giis.qacover.core.services;
+
+import giis.tdrules.openapi.model.DbSchema;
+import giis.tdrules.openapi.model.SqlParametersBody;
+import giis.tdrules.openapi.model.SqlRules;
+import giis.tdrules.openapi.model.SqlRulesBody;
+import giis.tdrules.openapi.model.SqlTableListBody;
+
+/**
+ * Temporal implementation of the client api, to be moved to tdrules-client
+ */
+public class TdRulesApi extends giis.tdrules.client.TdRulesApi {
+
+	private boolean useCache;
+	private String cacheLocation;
+
+	public TdRulesApi(String uri) {
+		super(uri);
+		this.cacheLocation = Configuration.getInstance().getCacheRulesLocation();
+		this.useCache = !"".equals(coalesce(cacheLocation));
+	}
+
+	/**
+	 * Gets the fpc rules for a query executed under the specified schema
+	 */
+	@Override
+	public SqlRules getRules(DbSchema schema, String query, String options) {
+		SqlRulesBody request=new SqlRulesBody().schema(schema).sql(query).options(coalesce(options));
+		TdRulesCache cache = getCache("rulesPost", request);
+
+		if (useCache() && cache.hit())
+			return (SqlRules)cache.getPayload(SqlRules.class);
+		SqlRules result = super.rulesPost(request);
+		if (useCache())
+			cache.putPayload(result);
+		return result;
+	}
+
+	public SqlTableListBody getTables(String sql) {
+		TdRulesCache cache = getCache("sqlTablesPost", sql);
+		if (useCache() && cache.hit())
+			return (SqlTableListBody)cache.getPayload(SqlTableListBody.class);
+		SqlTableListBody result = super.sqlTablesPost(sql);
+		if (useCache())
+			cache.putPayload(result);
+		return result;
+	}
+
+	public SqlParametersBody getParameters(String sql) {
+		TdRulesCache cache = getCache("sqlParametersPost", sql);
+		if (useCache() && cache.hit())
+			return (SqlParametersBody)cache.getPayload(SqlParametersBody.class);
+		SqlParametersBody result = super.sqlParametersPost(sql);
+		if (useCache())
+			cache.putPayload(result);
+		return result;
+	}
+
+	// Cache management
+
+	private boolean useCache() {
+		return this.useCache;
+	}
+	private TdRulesCache getCache(String endpoint, Object request) {
+		return useCache() ? new TdRulesCache(this.cacheLocation, endpoint, request) : null;
+	}
+	private String coalesce(String value) {
+		return value == null ? "" : value;
+	}
+
+}

qacover-core/src/main/java/giis/qacover/core/services/TdRulesCache.java

@@ -0,0 +1,59 @@
+package giis.qacover.core.services;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import giis.portable.util.FileUtil;
+import giis.portable.util.JavaCs;
+import giis.tdrules.model.io.ModelJsonSerializer;
+
+/**
+ * Temporal implementation of a local storage cache for payloads sent to
+ * TdRules, to be moved to tdrules-client
+ */
+public class TdRulesCache {
+	private static final Logger log = LoggerFactory.getLogger(TdRulesCache.class);
+
+	private ModelJsonSerializer serializer;
+	String endpoint;
+	String payload;
+	String hash;
+	String cacheFile;
+	String hit;
+
+	public TdRulesCache(String cacheFolder, String endpoint, Object request) {
+		serializer = new ModelJsonSerializer();
+		this.endpoint = endpoint;
+		this.payload = serializer.serialize(request, true);
+		this.hash = JavaCs.getHash(payload);
+		this.cacheFile = getCacheFile(cacheFolder, endpoint, hash);
+		this.hit = FileUtil.fileRead(cacheFile, false);
+		log.debug("Cache {} {} hit: {}", endpoint, hash, this.hit != null);
+	}
+
+	/**
+	 * Gets the payload of a given request from cache, if does not exists returns
+	 * null
+	 */
+	public boolean hit() {
+		return this.hit != null;
+	}
+
+	@SuppressWarnings("rawtypes")
+	public Object getPayload(Class clazz) {
+		return serializer.deserialize(hit, clazz);
+	}
+
+	/**
+	 * Saves the response payload of a given request to cache
+	 */
+	public void putPayload(Object result) {
+		FileUtil.fileWrite(cacheFile, serializer.serialize(result, true));
+		log.debug("Cache {} {} update: {}", endpoint, hash, result);
+	}
+
+	private String getCacheFile(String cacheFolder, String endpoint, String hash) {
+		return FileUtil.getPath(cacheFolder, endpoint, hash + ".json");
+	}
+
+}

qacover-core/src/test/java/test4giis/qacover/Base.java

@@ -57,11 +57,20 @@ public void setUp() throws SQLException {
 		variant = getVariant();
 		log.info("*** CURRENT DBMS - " + variant.getSgbdName());
 		rs = null;
-		options = Configuration.getInstance().reset().setName("qacovertest");
+		options = configureTestOptions();
 		new StoreService(options).dropRules().dropLast();
 		QueryStatement.setFaultInjector(null);
 		Locale.setDefault(new Locale("en", "US"));
 	}
+
+	/**
+	 * Default configuration for tests
+	 */
+	public static Configuration configureTestOptions() {
+		return Configuration.getInstance().reset()
+				.setName("qacovertest")
+				.setCacheRulesLocation(".tdrules-cache");
+	}
 
 	@After
 	public void tearDown() throws SQLException {

qacover-core/src/test/java/test4giis/qacover/TestEvaluation.java

@@ -94,7 +94,7 @@ public void testEvalInferParametersNegative() throws SQLException {
 
 	@Test
 	public void testEvalFpcOptions() throws SQLException {
-		Configuration.getInstance().reset().setFpcServiceOptions("noboundaries");
+		configureTestOptions().setFpcServiceOptions("noboundaries");
 		rs = app.queryNoParameters1Condition(-1);
 		assertEvalResults("select id,num,text from test where num>=-1", 
 				"1 0 abc\n2 99 xyz\n3 99 NULL", SqlUtil.resultSet2csv(rs," "), 
@@ -154,7 +154,7 @@ public void testEvalNoConditions() throws SQLException {
 
 	@Test
 	public void testEvalNoConditionsInferParams() throws SQLException {
-		Configuration.getInstance().reset().setInferQueryParameters(true);
+		Base.configureTestOptions().setInferQueryParameters(true);
 		rs = app.queryNoConditions();
 		assertEvalResults("SELECT id , num , text FROM test", "1 0 abc\n2 99 xyz\n3 99 NULL", SqlUtil.resultSet2csv(rs," "), "");
 	}

qacover-core/src/test/java/test4giis/qacover/TestFaults.java

@@ -72,7 +72,8 @@ private boolean contains(String text, String substring) {
 
 	@Test
 	public void testFaultConnectingService() throws SQLException {
-		options.setFpcServiceUrl("http://giis.uniovi.es/noexiste.xml");
+		options.setFpcServiceUrl("http://giis.uniovi.es/noexiste.xml")
+			.setCacheRulesLocation(""); // disable cache to run the actual service
 		rs=app.executeQuery("select id,num,text from test where num<9");
 		assertExceptionMessage(new Variability().isJava()
 				? "Error at Get query table names: ApiException"

qacover-core/src/test/java/test4giis/qacover/TestStore.java

@@ -228,29 +228,29 @@ public void testSameQueriesDifferentLine() throws SQLException {
 	@Test
 	public void testAbortByTableExclusion() throws SQLException {
 		// full match (implies an abort on the rule evaluation)
-		Configuration.getInstance().reset().setFpcServiceOptions("noboundaries").addTableExclusionExact("test");
+		super.configureTestOptions().setFpcServiceOptions("noboundaries").addTableExclusionExact("test");
 		rs = app.queryNoParameters1Condition(-1);
 		assertAbort("{}");
 		// approximate match (no abort)
-		Configuration.getInstance().reset().setFpcServiceOptions("noboundaries").addTableExclusionExact("tes");
+		super.configureTestOptions().setFpcServiceOptions("noboundaries").addTableExclusionExact("tes");
 		rs = app.queryNoParameters1Condition(-1);
 		assertNoAbort();
 	}
 
 	@Test
 	public void testAbortByClassExclusion() throws SQLException {
 		// full match (abort)
-		Configuration.getInstance().reset().setFpcServiceOptions("noboundaries")
+		super.configureTestOptions().setFpcServiceOptions("noboundaries")
 				.addClassExclusion("test4giis.qacoverapp.AppSimpleJdbc");
 		rs = app.queryNoParameters1Condition(-1);
 		assertAbort("");
 		// partial match
-		Configuration.getInstance().reset().setFpcServiceOptions("noboundaries")
+		super.configureTestOptions().setFpcServiceOptions("noboundaries")
 				.addClassExclusion("test4giis.qacoverapp.AppSimpleJdb");
 		rs = app.queryNoParameters1Condition(-1);
 		assertAbort("");
 		// no match, normal processing
-		Configuration.getInstance().reset().setFpcServiceOptions("noboundaries")
+		super.configureTestOptions().setFpcServiceOptions("noboundaries")
 				.addClassExclusion("test4giis.qacoverapp.AppSimpleJdbx");
 		rs = app.queryNoParameters1Condition(-1);
 		assertNoAbort();

qacover-core/src/test/java/test4giis/qacover/model/TestUtil.java

@@ -1,5 +1,6 @@
 package test4giis.qacover.model;
 
+import test4giis.qacover.Base;
 import test4giis.qacoverapp.AppSimpleJdbc;
 
 import static org.junit.Assert.assertEquals;
@@ -24,12 +25,12 @@ public class TestUtil {
 	// ensures clean start without custom options
 	@Before
 	public void setUp() {
-		Configuration.getInstance().reset();
+		Base.configureTestOptions();
 	}
 
 	@After
 	public void tearDown() {
-		Configuration.getInstance().reset();
+		Base.configureTestOptions();
 	}
 
 	@Test

qacover-core/src/test/java/test4giis/qacoversample/BaseTest.java

@@ -9,6 +9,7 @@
 
 import giis.qacover.core.services.Configuration;
 import giis.qacover.core.services.StoreService;
+import test4giis.qacover.Base;
 
 import java.sql.*;
 
@@ -51,7 +52,7 @@ public void tearDown() throws SQLException {
 	@BeforeClass
 	public static void classSetUp() {
 		log.info("Reset coverage parameters");
-		Configuration options = Configuration.getInstance().reset().setName("qacoversample")
+		Configuration options = Base.configureTestOptions().setName("qacoversample")
 				.setFpcServiceOptions("noboundaries notautology");
 		// instantiates an store to reset rules
 		new StoreService(options).dropRules().dropLast();