Add more integ tests for query insights (#71)

Signed-off-by: Chenyang Ji <cyji@amazon.com>

build.gradle

@@ -263,6 +263,7 @@ integTest {
   if (System.getProperty("security.enabled") == "true" || System.getProperty("https") == "true") {
     // Exclude this IT, because they executed in another task (:integTestWithSecurity)
     exclude 'org/opensearch/plugin/insights/rules/resthandler/top_queries/TopQueriesRestIT.class'
+    exclude 'org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterIT.class'
   }
 }
 

gradle/wrapper/gradle-wrapper.properties

@@ -8,7 +8,6 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961

src/test/java/org/opensearch/plugin/insights/QueryInsightsRestTestCase.java

@@ -0,0 +1,216 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugin.insights;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager;
+import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder;
+import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;
+import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
+import org.apache.hc.core5.http.Header;
+import org.apache.hc.core5.http.HttpHost;
+import org.apache.hc.core5.http.message.BasicHeader;
+import org.apache.hc.core5.http.nio.ssl.TlsStrategy;
+import org.apache.hc.core5.reactor.ssl.TlsDetails;
+import org.apache.hc.core5.ssl.SSLContextBuilder;
+import org.apache.hc.core5.util.Timeout;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.client.RestClient;
+import org.opensearch.client.RestClientBuilder;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.MediaType;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.test.rest.OpenSearchRestTestCase;
+
+public abstract class QueryInsightsRestTestCase extends OpenSearchRestTestCase {
+    protected static final String QUERY_INSIGHTS_INDICES_PREFIX = "top_queries";
+
+    protected boolean isHttps() {
+        return Optional.ofNullable(System.getProperty("https")).map("true"::equalsIgnoreCase).orElse(false);
+    }
+
+    @Override
+    protected String getProtocol() {
+        return isHttps() ? "https" : "http";
+    }
+
+    @Override
+    protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
+        RestClientBuilder builder = RestClient.builder(hosts);
+        if (isHttps()) {
+            configureHttpsClient(builder, settings);
+        } else {
+            configureClient(builder, settings);
+        }
+
+        builder.setStrictDeprecationMode(false);
+        return builder.build();
+    }
+
+    protected static void configureClient(RestClientBuilder builder, Settings settings) throws IOException {
+        String userName = System.getProperty("user");
+        String password = System.getProperty("password");
+        if (userName != null && password != null) {
+            builder.setHttpClientConfigCallback(httpClientBuilder -> {
+                BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+                credentialsProvider.setCredentials(
+                    new AuthScope(null, -1),
+                    new UsernamePasswordCredentials(userName, password.toCharArray())
+                );
+                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+            });
+        }
+        OpenSearchRestTestCase.configureClient(builder, settings);
+    }
+
+    protected static void configureHttpsClient(RestClientBuilder builder, Settings settings) throws IOException {
+        // Similar to client configuration with OpenSearch:
+        // https://github.com/opensearch-project/OpenSearch/blob/2.15.1/test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java#L841-L863
+        builder.setHttpClientConfigCallback(httpClientBuilder -> {
+            String userName = Optional.ofNullable(System.getProperty("user"))
+                .orElseThrow(() -> new RuntimeException("user name is missing"));
+            String password = Optional.ofNullable(System.getProperty("password"))
+                .orElseThrow(() -> new RuntimeException("password is missing"));
+            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            final AuthScope anyScope = new AuthScope(null, -1);
+            credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(userName, password.toCharArray()));
+            try {
+                final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create()
+                    .setHostnameVerifier(NoopHostnameVerifier.INSTANCE)
+                    .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build())
+                    // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219
+                    .setTlsDetailsFactory(sslEngine -> new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()))
+                    .build();
+                final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create()
+                    .setTlsStrategy(tlsStrategy)
+                    .build();
+                return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager);
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+        Map<String, String> headers = ThreadContext.buildDefaultHeaders(settings);
+        Header[] defaultHeaders = new Header[headers.size()];
+        int i = 0;
+        for (Map.Entry<String, String> entry : headers.entrySet()) {
+            defaultHeaders[i++] = new BasicHeader(entry.getKey(), entry.getValue());
+        }
+        builder.setDefaultHeaders(defaultHeaders);
+        final String socketTimeoutString = settings.get(CLIENT_SOCKET_TIMEOUT);
+        final TimeValue socketTimeout = TimeValue.parseTimeValue(
+            socketTimeoutString == null ? "60s" : socketTimeoutString,
+            CLIENT_SOCKET_TIMEOUT
+        );
+        builder.setRequestConfigCallback(
+            conf -> conf.setResponseTimeout(Timeout.ofMilliseconds(Math.toIntExact(socketTimeout.getMillis())))
+        );
+        if (settings.hasValue(CLIENT_PATH_PREFIX)) {
+            builder.setPathPrefix(settings.get(CLIENT_PATH_PREFIX));
+        }
+    }
+
+    /**
+     * wipeAllIndices won't work since it cannot delete security index. Use
+     * wipeAllQueryInsightsIndices instead.
+     */
+    @Override
+    protected boolean preserveIndicesUponCompletion() {
+        return true;
+    }
+
+    @Before
+    public void runBeforeEachTest() throws IOException {
+        // Create documents for search
+        Request request = new Request("POST", "/my-index-0/_doc");
+        request.setJsonEntity(createDocumentsBody());
+        Response response = client().performRequest(request);
+
+        Assert.assertEquals(201, response.getStatusLine().getStatusCode());
+    }
+
+    @SuppressWarnings("unchecked")
+    @After
+    public void wipeAllQueryInsightsIndices() throws Exception {
+        Response response = adminClient().performRequest(new Request("GET", "/_cat/indices?format=json&expand_wildcards=all"));
+        MediaType mediaType = MediaType.fromMediaType(response.getEntity().getContentType());
+        try (
+            XContentParser parser = mediaType.xContent()
+                .createParser(
+                    NamedXContentRegistry.EMPTY,
+                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                    response.getEntity().getContent()
+                )
+        ) {
+            XContentParser.Token token = parser.nextToken();
+            List<Map<String, Object>> parserList = null;
+            if (token == XContentParser.Token.START_ARRAY) {
+                parserList = parser.listOrderedMap().stream().map(obj -> (Map<String, Object>) obj).collect(Collectors.toList());
+            } else {
+                parserList = Collections.singletonList(parser.mapOrdered());
+            }
+
+            for (Map<String, Object> index : parserList) {
+                final String indexName = (String) index.get("index");
+                if (indexName.startsWith(QUERY_INSIGHTS_INDICES_PREFIX)) {
+                    adminClient().performRequest(new Request("DELETE", "/" + indexName));
+                }
+            }
+        }
+    }
+
+    protected String defaultTopQueriesSettings() {
+        return "{\n"
+            + "    \"persistent\" : {\n"
+            + "        \"search.insights.top_queries.latency.enabled\" : \"true\",\n"
+            + "        \"search.insights.top_queries.latency.window_size\" : \"600s\",\n"
+            + "        \"search.insights.top_queries.latency.top_n_size\" : 5\n"
+            + "    }\n"
+            + "}";
+    }
+
+    protected String createDocumentsBody() {
+        return "{\n"
+            + "  \"@timestamp\": \"2099-11-15T13:12:00\",\n"
+            + "  \"message\": \"this is document 1\",\n"
+            + "  \"user\": {\n"
+            + "    \"id\": \"cyji\"\n"
+            + "  }\n"
+            + "}";
+    }
+
+    protected String searchBody() {
+        return "{}";
+    }
+
+    protected void doSearch(int times) throws IOException {
+        for (int i = 0; i < times; i++) {
+            // Do Search
+            Request request = new Request("GET", "/my-index-0/_search?size=20&pretty");
+            request.setJsonEntity(searchBody());
+            Response response = client().performRequest(request);
+            Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+        }
+    }
+}

src/test/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterIT.java

@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugin.insights.core.exporter;
+
+import java.io.IOException;
+import org.junit.Assert;
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+import org.opensearch.client.ResponseException;
+import org.opensearch.plugin.insights.QueryInsightsRestTestCase;
+
+/** Rest Action tests for query  */
+public class QueryInsightsExporterIT extends QueryInsightsRestTestCase {
+    /**
+     * Test Top Queries setting endpoints
+     *
+     * @throws IOException IOException
+     */
+    public void testQueryInsightsExporterSettings() throws IOException {
+        // test invalid settings
+        for (String setting : invalidExporterSettings()) {
+            Request request = new Request("PUT", "/_cluster/settings");
+            request.setJsonEntity(setting);
+            try {
+                client().performRequest(request);
+                fail("Should not succeed with invalid exporter settings");
+            } catch (ResponseException e) {
+                assertEquals(400, e.getResponse().getStatusLine().getStatusCode());
+            }
+        }
+
+        // Test enable Top N Queries feature
+        Request request = new Request("PUT", "/_cluster/settings");
+        request.setJsonEntity(defaultExporterSettings());
+        Response response = client().performRequest(request);
+        Assert.assertEquals(200, response.getStatusLine().getStatusCode());
+    }
+
+    private String defaultExporterSettings() {
+        return "{\n"
+            + "    \"persistent\" : {\n"
+            + "        \"search.insights.top_queries.latency.exporter.config.index\" : \"YYYY.MM.dd\",\n"
+            + "        \"search.insights.top_queries.latency.exporter.type\" : \"local_index\"\n"
+            + "    }\n"
+            + "}";
+    }
+
+    private String[] invalidExporterSettings() {
+        return new String[] {
+            "{\n"
+                + "    \"persistent\" : {\n"
+                + "        \"search.insights.top_queries.latency.exporter.type\" : invalid_type\n"
+                + "    }\n"
+                + "}",
+            "{\n"
+                + "    \"persistent\" : {\n"
+                + "        \"search.insights.top_queries.latency.exporter.type\" : local_index,\n"
+                + "        \"search.insights.top_queries.latency.exporter.config.index\" : \"1a2b\"\n"
+                + "    }\n"
+                + "}" };
+    }
+}