diff --git a/.idea/copyright/SPDX_ALv2.xml b/.idea/copyright/SPDX_ALv2.xml index a8bdf69c..3c6f9c76 100644 --- a/.idea/copyright/SPDX_ALv2.xml +++ b/.idea/copyright/SPDX_ALv2.xml @@ -1,6 +1,6 @@ - - + + \ No newline at end of file diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/IndexManager.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/IndexManager.java index 1ad06bf1..2e6c28d3 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/IndexManager.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/IndexManager.java @@ -24,7 +24,6 @@ public class IndexManager { public static final String FIELD_TYPE_KEY = "type"; public static final String MAPPING_PROPERTIES_KEY = "properties"; - public static final String DOCUMENT_TYPE = "_doc"; private static final Logger LOGGER = LogManager.getLogger(IndexManager.class); private final IndicesAdminClient client; diff --git a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONAction.java b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONAction.java index 567ab0b8..d4eba8ed 100644 --- a/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONAction.java +++ b/src/main/java/org/opensearch/geospatial/action/upload/geojson/UploadGeoJSONAction.java @@ -15,7 +15,7 @@ public class UploadGeoJSONAction extends ActionType { - public static UploadGeoJSONAction INSTANCE = new UploadGeoJSONAction(); + public static final UploadGeoJSONAction INSTANCE = new UploadGeoJSONAction(); public static final String NAME = "cluster:admin/upload_geojson_action"; private UploadGeoJSONAction() { diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index d3e483ed..7eac0604 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -5,31 +5,44 @@ package org.opensearch.geospatial.plugin; -import static java.util.Collections.singletonList; - +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Supplier; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionResponse; +import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.collect.MapBuilder; +import org.opensearch.common.io.stream.NamedWriteableRegistry; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.env.Environment; +import org.opensearch.env.NodeEnvironment; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONTransportAction; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; +import org.opensearch.geospatial.stats.upload.RestUploadStatsAction; +import org.opensearch.geospatial.stats.upload.UploadStats; +import org.opensearch.geospatial.stats.upload.UploadStatsAction; +import org.opensearch.geospatial.stats.upload.UploadStatsTransportAction; import org.opensearch.ingest.Processor; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IngestPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.repositories.RepositoriesService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; +import org.opensearch.script.ScriptService; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.watcher.ResourceWatcherService; /** * Entry point for Geospatial features. It provides additional Processors, Actions @@ -44,6 +57,23 @@ public Map getProcessors(Processor.Parameters paramet .immutableMap(); } + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier + ) { + return List.of(UploadStats.getInstance()); + } + @Override public List getRestHandlers( Settings settings, @@ -55,11 +85,15 @@ public List getRestHandlers( Supplier nodesInCluster ) { RestUploadGeoJSONAction uploadGeoJSONAction = new RestUploadGeoJSONAction(); - return singletonList(uploadGeoJSONAction); + RestUploadStatsAction statsAction = new RestUploadStatsAction(); + return List.of(statsAction, uploadGeoJSONAction); } @Override public List> getActions() { - return singletonList(new ActionHandler<>(UploadGeoJSONAction.INSTANCE, UploadGeoJSONTransportAction.class)); + return List.of( + new ActionHandler<>(UploadGeoJSONAction.INSTANCE, UploadGeoJSONTransportAction.class), + new ActionHandler<>(UploadStatsAction.INSTANCE, UploadStatsTransportAction.class) + ); } } diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/RestUploadStatsAction.java b/src/main/java/org/opensearch/geospatial/stats/upload/RestUploadStatsAction.java new file mode 100644 index 00000000..93818989 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/stats/upload/RestUploadStatsAction.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; +import static org.opensearch.rest.RestRequest.Method.GET; + +import java.util.List; + +import org.opensearch.client.node.NodeClient; +import org.opensearch.rest.BaseRestHandler; +import org.opensearch.rest.RestRequest; +import org.opensearch.rest.action.RestToXContentListener; + +public class RestUploadStatsAction extends BaseRestHandler { + + private static final String NAME = "upload_stats"; + public static final String ACTION_OBJECT = "_upload"; + + public static final String ACTION_STATS = "stats"; + + @Override + public String getName() { + return NAME; + } + + @Override + public List routes() { + String path = String.join(URL_DELIMITER, getPluginURLPrefix(), ACTION_OBJECT, ACTION_STATS); + return List.of(new Route(GET, path)); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) { + return channel -> nodeClient.execute(UploadStatsAction.INSTANCE, new UploadStatsRequest(), new RestToXContentListener<>(channel)); + } +} diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java index a41389d5..a2a134cb 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStats.java @@ -58,6 +58,7 @@ public static UploadStats getInstance() { * @throws IOException if cannot read {@link UploadStats} from given input */ public static UploadStats fromStreamInput(StreamInput input) throws IOException { + Objects.requireNonNull(input, "StreamInput cannot be null"); UploadStats instance = new UploadStats(); instance.totalAPICount.inc(input.readVLong()); instance.metrics.addAll(input.readSet(UploadMetric.UploadMetricBuilder::fromStreamInput)); diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsAction.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsAction.java new file mode 100644 index 00000000..d51bb7db --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsAction.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import org.opensearch.action.ActionType; + +public class UploadStatsAction extends ActionType { + + public static final UploadStatsAction INSTANCE = new UploadStatsAction(); + public static final String NAME = "cluster:admin/geospatial/stats"; + + public UploadStatsAction() { + super(NAME, UploadStatsResponse::new); + } +} diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeRequest.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeRequest.java new file mode 100644 index 00000000..5ddb0e54 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeRequest.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import java.io.IOException; + +import org.opensearch.action.support.nodes.BaseNodeRequest; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; + +public class UploadStatsNodeRequest extends BaseNodeRequest { + + private final UploadStatsRequest request; + + public UploadStatsNodeRequest(StreamInput in) throws IOException { + super(in); + request = new UploadStatsRequest(in); + } + + public UploadStatsNodeRequest(UploadStatsRequest request) { + this.request = request; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + request.writeTo(out); + } + +} diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponse.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponse.java new file mode 100644 index 00000000..c5801cd9 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponse.java @@ -0,0 +1,49 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import java.io.IOException; +import java.util.Objects; + +import org.opensearch.action.support.nodes.BaseNodeResponse; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.xcontent.ToXContentObject; +import org.opensearch.common.xcontent.XContentBuilder; + +public class UploadStatsNodeResponse extends BaseNodeResponse implements ToXContentObject { + + private static final String UPLOADS = "uploads"; + private final UploadStats uploadStats; + + public UploadStatsNodeResponse(DiscoveryNode node, UploadStats uploadStats) { + super(node); + this.uploadStats = Objects.requireNonNull(uploadStats, "upload stats cannot be null"); + } + + public UploadStatsNodeResponse(StreamInput in) throws IOException { + super(in); + uploadStats = UploadStats.fromStreamInput(in); + } + + public UploadStats getUploadStats() { + return uploadStats; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + uploadStats.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(UPLOADS); + uploadStats.toXContent(builder, params); + return builder.endObject(); + } +} diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsRequest.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsRequest.java new file mode 100644 index 00000000..9bcacc99 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsRequest.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import java.io.IOException; + +import org.opensearch.action.support.nodes.BaseNodesRequest; +import org.opensearch.common.io.stream.StreamInput; + +public class UploadStatsRequest extends BaseNodesRequest { + + /** + * Empty constructor needed for UploadStatsTransportAction + */ + public UploadStatsRequest() { + super((String[]) null); + } + + protected UploadStatsRequest(StreamInput in) throws IOException { + super(in); + } +} diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsResponse.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsResponse.java new file mode 100644 index 00000000..fd870193 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsResponse.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.nodes.BaseNodesResponse; +import org.opensearch.cluster.ClusterName; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.common.xcontent.ToXContentObject; +import org.opensearch.common.xcontent.XContentBuilder; + +public class UploadStatsResponse extends BaseNodesResponse implements Writeable, ToXContentObject { + + public UploadStatsResponse(StreamInput in) throws IOException { + super(new ClusterName(in), in.readList(UploadStatsNodeResponse::new), in.readList(FailedNodeException::new)); + } + + public UploadStatsResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(UploadStatsNodeResponse::new); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodeResponses) throws IOException { + out.writeList(nodeResponses); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + final Map nodeIDStatsMap = getNodes().stream() + .collect(Collectors.toMap(response -> response.getNode().getId(), UploadStatsNodeResponse::getUploadStats)); + UploadStatsService uploadStatsService = new UploadStatsService(nodeIDStatsMap); + return uploadStatsService.toXContent(builder, params); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UploadStatsResponse otherResponse = (UploadStatsResponse) o; + return Objects.equals(getNodes(), otherResponse.getNodes()) && Objects.equals(failures(), otherResponse.failures()); + } + + @Override + public int hashCode() { + return Objects.hash(getNodes(), failures()); + } + +} diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsService.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsService.java index 20f8b1d8..4657526b 100644 --- a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsService.java +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsService.java @@ -16,11 +16,9 @@ import org.opensearch.common.xcontent.ToXContentFragment; import org.opensearch.common.xcontent.XContentBuilder; -// Service to calculate summary of upload stats and generate XContent for StatsResponse +// Service to calculate summary of upload stats and generate XContent for UploadStatsResponse public class UploadStatsService implements ToXContentFragment { - public static final String UPLOADS = "uploads"; - public static final String TOTAL = "total"; public static final String METRICS = "metrics"; public static final String NODE_ID = "node_id"; private final Map uploadStats; @@ -35,7 +33,6 @@ public UploadStatsService(Map uploadStats) { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { /* { - "uploads": { "total": { request_count : # of request, "upload" : sum of documents to upload across API, @@ -53,10 +50,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws "duration" : duration in milliseconds to ingest document }, ...... ] - } } */ - builder.startObject(UPLOADS); + builder.startObject(); totalUploadStats.toXContent(builder, params); builder.startArray(METRICS); if (totalUploadStats.isUploadStatsEmpty()) { diff --git a/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsTransportAction.java b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsTransportAction.java new file mode 100644 index 00000000..5caa3ac0 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/stats/upload/UploadStatsTransportAction.java @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import java.io.IOException; +import java.util.List; + +import org.opensearch.action.FailedNodeException; +import org.opensearch.action.support.ActionFilters; +import org.opensearch.action.support.nodes.TransportNodesAction; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.threadpool.ThreadPool; +import org.opensearch.transport.TransportService; + +public class UploadStatsTransportAction extends TransportNodesAction< + UploadStatsRequest, + UploadStatsResponse, + UploadStatsNodeRequest, + UploadStatsNodeResponse> { + + private final TransportService transportService; + private final UploadStats uploadStats; + + @Inject + public UploadStatsTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + UploadStats uploadStats + ) { + super( + UploadStatsAction.NAME, + threadPool, + clusterService, + transportService, + actionFilters, + UploadStatsRequest::new, + UploadStatsNodeRequest::new, + ThreadPool.Names.MANAGEMENT, + UploadStatsNodeResponse.class + ); + this.transportService = transportService; + this.uploadStats = uploadStats; + } + + @Override + protected UploadStatsResponse newResponse( + UploadStatsRequest nodesRequest, + List nodeResponses, + List failures + ) { + return new UploadStatsResponse(clusterService.getClusterName(), nodeResponses, failures); + } + + @Override + protected UploadStatsNodeRequest newNodeRequest(UploadStatsRequest nodesRequest) { + return new UploadStatsNodeRequest(nodesRequest); + } + + @Override + protected UploadStatsNodeResponse newNodeResponse(StreamInput streamInput) throws IOException { + return new UploadStatsNodeResponse(streamInput); + } + + @Override + protected UploadStatsNodeResponse nodeOperation(UploadStatsNodeRequest nodeRequest) { + return new UploadStatsNodeResponse(transportService.getLocalNode(), uploadStats); + } +} diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index 16139d68..1d98a7db 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -8,8 +8,9 @@ import static java.util.stream.Collectors.joining; import static org.opensearch.geospatial.GeospatialObjectBuilder.buildProperties; import static org.opensearch.geospatial.GeospatialObjectBuilder.randomGeoJSONFeature; -import static org.opensearch.geospatial.GeospatialTestHelper.*; +import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.FIELD_DATA; +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; import java.io.IOException; import java.util.Collections; @@ -33,6 +34,7 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent; import org.opensearch.geospatial.processor.FeatureProcessor; +import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; import org.opensearch.ingest.Pipeline; import org.opensearch.rest.RestStatus; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -173,4 +175,29 @@ protected int getIndexDocumentCount(String index) throws IOException { MatcherAssert.assertThat(FIELD_COUNT_KEY + " does not exist", responseMap, Matchers.hasKey(FIELD_COUNT_KEY)); return (Integer) responseMap.get(FIELD_COUNT_KEY); } + + private Response uploadGeoJSONFeaturesByMethod(String method, int featureCount, String indexName, String geospatialFieldName) + throws IOException { + // upload geoJSON + String path = String.join( + URL_DELIMITER, + getPluginURLPrefix(), + RestUploadGeoJSONAction.ACTION_OBJECT, + RestUploadGeoJSONAction.ACTION_UPLOAD + ); + Request request = new Request(method, path); + final JSONObject requestBody = buildUploadGeoJSONRequestContent(featureCount, indexName, geospatialFieldName); + request.setJsonEntity(requestBody.toString()); + return client().performRequest(request); + } + + protected final Response uploadGeoJSONFeaturesIntoExistingIndex(int featureCount, String indexName, String geospatialFieldName) + throws IOException { + return uploadGeoJSONFeaturesByMethod("PUT", featureCount, indexName, geospatialFieldName); + } + + protected final Response uploadGeoJSONFeatures(int featureCount, String indexName, String geospatialFieldName) throws IOException { + return uploadGeoJSONFeaturesByMethod("POST", featureCount, indexName, geospatialFieldName); + } + } diff --git a/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java b/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java index cee232c9..8c57fe6b 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialTestHelper.java @@ -12,6 +12,8 @@ package org.opensearch.geospatial; import static org.apache.lucene.tests.util.LuceneTestCase.random; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.opensearch.geospatial.GeospatialObjectBuilder.buildProperties; import static org.opensearch.geospatial.GeospatialObjectBuilder.randomGeoJSONFeature; import static org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent.FIELD_DATA; @@ -146,4 +148,10 @@ public static StringBuilder buildFieldNameValuePair(Object field, Object value) return builder.append("\"").append(value).append("\""); } + public static String removeStartAndEndObject(String content) { + assertNotNull(content); + assertTrue("content length should be at least 2", content.length() > 1); + return content.substring(1, content.length() - 1); + } + } diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index fb0e4d08..1a775955 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -5,19 +5,16 @@ package org.opensearch.geospatial.plugin; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.isA; - import java.util.List; import java.util.Map; -import org.hamcrest.MatcherAssert; import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionResponse; import org.opensearch.common.settings.Settings; import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction; import org.opensearch.geospatial.processor.FeatureProcessor; import org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction; +import org.opensearch.geospatial.stats.upload.RestUploadStatsAction; import org.opensearch.ingest.Processor; import org.opensearch.plugins.ActionPlugin; import org.opensearch.plugins.IngestPlugin; @@ -26,6 +23,8 @@ public class GeospatialPluginTests extends OpenSearchTestCase { + private final List SUPPORTED_REST_HANDLERS = List.of(new RestUploadGeoJSONAction(), new RestUploadStatsAction()); + public void testIsAnIngestPlugin() { GeospatialPlugin plugin = new GeospatialPlugin(); assertTrue(plugin instanceof IngestPlugin); @@ -40,13 +39,7 @@ public void testFeatureProcessorIsAdded() { public void testTotalRestHandlers() { GeospatialPlugin plugin = new GeospatialPlugin(); - assertEquals(1, plugin.getRestHandlers(Settings.EMPTY, null, null, null, null, null, null).size()); - } - - public void testUploadGeoJSONRestHandlerIsAdded() { - GeospatialPlugin plugin = new GeospatialPlugin(); - List restHandlers = plugin.getRestHandlers(Settings.EMPTY, null, null, null, null, null, null); - MatcherAssert.assertThat("should contain RestUploadGeoJSONAction", restHandlers, contains(isA(RestUploadGeoJSONAction.class))); + assertEquals(SUPPORTED_REST_HANDLERS.size(), plugin.getRestHandlers(Settings.EMPTY, null, null, null, null, null, null).size()); } public void testUploadGeoJSONTransportIsAdded() { diff --git a/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java b/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java index 44a8c74c..d0792b9e 100644 --- a/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java +++ b/src/test/java/org/opensearch/geospatial/rest/action/upload/geojson/RestUploadGeoJSONActionIT.java @@ -12,21 +12,15 @@ package org.opensearch.geospatial.rest.action.upload.geojson; import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; -import static org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction.ACTION_OBJECT; -import static org.opensearch.geospatial.rest.action.upload.geojson.RestUploadGeoJSONAction.ACTION_UPLOAD; -import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import org.json.JSONObject; -import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.settings.Settings; import org.opensearch.geospatial.GeospatialRestTestCase; -import org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequestContent; import org.opensearch.rest.RestStatus; public class RestUploadGeoJSONActionIT extends GeospatialRestTestCase { @@ -35,13 +29,9 @@ public class RestUploadGeoJSONActionIT extends GeospatialRestTestCase { public void testGeoJSONUploadSuccessPostMethod() throws IOException { - String path = String.join(URL_DELIMITER, getPluginURLPrefix(), ACTION_OBJECT, ACTION_UPLOAD); - Request request = new Request("POST", path); - final JSONObject requestBody = buildUploadGeoJSONRequestContent(NUMBER_OF_FEATURES_TO_ADD, null, null); - final String index = requestBody.getString(UploadGeoJSONRequestContent.FIELD_INDEX.getPreferredName()); + final String index = randomLowerCaseString(); assertIndexNotExists(index); - request.setJsonEntity(requestBody.toString()); - Response response = client().performRequest(request); + Response response = uploadGeoJSONFeatures(NUMBER_OF_FEATURES_TO_ADD, index, null); assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); assertIndexExists(index); assertEquals("failed to index documents", NUMBER_OF_FEATURES_TO_ADD, getIndexDocumentCount(index)); @@ -55,23 +45,17 @@ public void testGeoJSONUploadFailIndexExists() throws IOException { geoFields.put(geoFieldName, "geo_shape"); createIndex(index, Settings.EMPTY, geoFields); assertIndexExists(index); - String path = String.join(URL_DELIMITER, getPluginURLPrefix(), ACTION_OBJECT, ACTION_UPLOAD); - Request request = new Request("POST", path); - final JSONObject requestBody = buildUploadGeoJSONRequestContent(NUMBER_OF_FEATURES_TO_ADD, index, geoFieldName); - request.setJsonEntity(requestBody.toString()); - final ResponseException responseException = assertThrows(ResponseException.class, () -> client().performRequest(request)); + final ResponseException responseException = assertThrows( + ResponseException.class, + () -> uploadGeoJSONFeatures(NUMBER_OF_FEATURES_TO_ADD, index, geoFieldName) + ); assertTrue("Not an expected exception", responseException.getMessage().contains("resource_already_exists_exception")); } public void testGeoJSONUploadSuccessPutMethod() throws IOException { - String path = String.join(URL_DELIMITER, getPluginURLPrefix(), ACTION_OBJECT, ACTION_UPLOAD); - Request request = new Request("PUT", path); - final JSONObject requestBody = buildUploadGeoJSONRequestContent(NUMBER_OF_FEATURES_TO_ADD, null, null); - final String index = requestBody.getString(UploadGeoJSONRequestContent.FIELD_INDEX.getPreferredName()); - assertIndexNotExists(index); - request.setJsonEntity(requestBody.toString()); - Response response = client().performRequest(request); + String index = randomLowerCaseString(); + Response response = uploadGeoJSONFeaturesIntoExistingIndex(NUMBER_OF_FEATURES_TO_ADD, index, null); assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); assertIndexExists(index); assertEquals("failed to index documents", NUMBER_OF_FEATURES_TO_ADD, getIndexDocumentCount(index)); @@ -81,17 +65,11 @@ public void testGeoJSONPutMethodUploadIndexExists() throws IOException { String index = randomLowerCaseString(); String geoFieldName = randomLowerCaseString(); - String path = String.join(URL_DELIMITER, getPluginURLPrefix(), ACTION_OBJECT, ACTION_UPLOAD); - Request request = new Request("PUT", path); - final JSONObject requestBody = buildUploadGeoJSONRequestContent(NUMBER_OF_FEATURES_TO_ADD, index, geoFieldName); - request.setJsonEntity(requestBody.toString()); - Response response = client().performRequest(request); + Response response = uploadGeoJSONFeaturesIntoExistingIndex(NUMBER_OF_FEATURES_TO_ADD, index, geoFieldName); assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); int indexDocumentCount = getIndexDocumentCount(index); // upload again - final JSONObject requestBodyUpload = buildUploadGeoJSONRequestContent(NUMBER_OF_FEATURES_TO_ADD, index, geoFieldName); - request.setJsonEntity(requestBodyUpload.toString()); - Response uploadGeoJSONSecondTimeResponse = client().performRequest(request); + Response uploadGeoJSONSecondTimeResponse = uploadGeoJSONFeaturesIntoExistingIndex(NUMBER_OF_FEATURES_TO_ADD, index, geoFieldName); assertEquals(RestStatus.OK, RestStatus.fromCode(uploadGeoJSONSecondTimeResponse.getStatusLine().getStatusCode())); int expectedDocCountAfterUpload = indexDocumentCount + NUMBER_OF_FEATURES_TO_ADD; assertEquals("failed to index documents", expectedDocCountAfterUpload, getIndexDocumentCount(index)); diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java b/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java new file mode 100644 index 00000000..e702d232 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/stats/upload/RestUploadStatsActionIT.java @@ -0,0 +1,55 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; +import static org.opensearch.geospatial.stats.upload.RestUploadStatsAction.ACTION_OBJECT; +import static org.opensearch.geospatial.stats.upload.RestUploadStatsAction.ACTION_STATS; + +import java.io.IOException; + +import org.apache.http.util.EntityUtils; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.geospatial.GeospatialRestTestCase; +import org.opensearch.rest.RestStatus; + +public class RestUploadStatsActionIT extends GeospatialRestTestCase { + + private static final int NUMBER_OF_FEATURES_TO_ADD = 3; + + private String getUploadStatsPath() { + return String.join(URL_DELIMITER, getPluginURLPrefix(), ACTION_OBJECT, ACTION_STATS); + } + + private String getStatsResponseAsString() throws IOException { + Request statsRequest = new Request("GET", getUploadStatsPath()); + Response statsResponse = client().performRequest(statsRequest); + return EntityUtils.toString(statsResponse.getEntity()); + } + + public void testStatsAPISuccess() throws IOException { + + Request request = new Request("GET", getUploadStatsPath()); + Response response = client().performRequest(request); + assertEquals("Failed to retrieve stats", RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + } + + public void testStatsAreUpdatedAfterUpload() throws IOException { + // get current stats response + final String currentUploadStats = getStatsResponseAsString(); + assertNotNull(currentUploadStats); + + Response response = uploadGeoJSONFeatures(NUMBER_OF_FEATURES_TO_ADD, null, null); + assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode())); + + // get stats response after an upload + final String newUploadStats = getStatsResponseAsString(); + assertNotNull(newUploadStats); + assertTrue("New metrics are not added", newUploadStats.length() > currentUploadStats.length()); + } + +} diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseBuilder.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseBuilder.java new file mode 100644 index 00000000..fd4fab04 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseBuilder.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; +import static org.opensearch.geospatial.stats.upload.UploadStatsBuilder.randomUploadStats; +import static org.opensearch.test.OpenSearchTestCase.buildNewFakeTransportAddress; +import static org.opensearch.test.OpenSearchTestCase.randomIntBetween; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.IntStream; + +import org.opensearch.Version; +import org.opensearch.cluster.node.DiscoveryNode; + +public class UploadStatsNodeResponseBuilder { + private static final int MIN_STATS_NODE_RESPONSE = 2; + private static final int MAX_STATS_NODE_RESPONSE = 5; + + public static UploadStatsNodeResponse randomStatsNodeResponse(String nodeID) { + DiscoveryNode node = new DiscoveryNode( + randomLowerCaseString(), + nodeID, + buildNewFakeTransportAddress(), + emptyMap(), + emptySet(), + Version.CURRENT + ); + return new UploadStatsNodeResponse(node, randomUploadStats()); + } + + public static Map randomStatsNodeResponse() { + int randomResponseCount = randomIntBetween(MIN_STATS_NODE_RESPONSE, MAX_STATS_NODE_RESPONSE); + Map nodesResponseMap = new HashMap<>(randomResponseCount); + IntStream.range(0, randomResponseCount).forEach(unused -> { + String nodeID = randomLowerCaseString(); + nodesResponseMap.put(nodeID, randomStatsNodeResponse(nodeID)); + }); + return nodesResponseMap; + } +} diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseTests.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseTests.java new file mode 100644 index 00000000..e453f03b --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsNodeResponseTests.java @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import java.io.IOException; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.test.OpenSearchTestCase; + +public class UploadStatsNodeResponseTests extends OpenSearchTestCase { + + public void testStream() throws IOException { + UploadStatsNodeResponse nodeResponse = UploadStatsNodeResponseBuilder.randomStatsNodeResponse( + GeospatialTestHelper.randomLowerCaseString() + ); + BytesStreamOutput output = new BytesStreamOutput(); + nodeResponse.writeTo(output); + StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + + UploadStatsNodeResponse serializedNodeResponse = new UploadStatsNodeResponse(in); + assertNotNull("serialized node response cannot be null", serializedNodeResponse); + assertArrayEquals( + "mismatch metrics during serialization", + nodeResponse.getUploadStats().getMetrics().toArray(), + serializedNodeResponse.getUploadStats().getMetrics().toArray() + ); + assertEquals( + "mismatch api count", + nodeResponse.getUploadStats().getTotalAPICount(), + serializedNodeResponse.getUploadStats().getTotalAPICount() + ); + } +} diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsResponseTests.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsResponseTests.java new file mode 100644 index 00000000..34077149 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsResponseTests.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.stats.upload; + +import static java.util.Collections.emptyList; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.geospatial.GeospatialTestHelper.randomLowerCaseString; +import static org.opensearch.geospatial.GeospatialTestHelper.removeStartAndEndObject; +import static org.opensearch.geospatial.stats.upload.UploadStatsNodeResponseBuilder.randomStatsNodeResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.opensearch.cluster.ClusterName; +import org.opensearch.common.Strings; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.test.OpenSearchTestCase; + +public class UploadStatsResponseTests extends OpenSearchTestCase { + + public void testXContentWithMetrics() throws IOException { + + Map nodeResponse = randomStatsNodeResponse(); + UploadStatsResponse uploadStatsResponse = new UploadStatsResponse( + new ClusterName(randomLowerCaseString()), + new ArrayList<>(nodeResponse.values()), + emptyList() + ); + final XContentBuilder serviceContentBuilder = jsonBuilder(); + uploadStatsResponse.toXContent(serviceContentBuilder, ToXContent.EMPTY_PARAMS); + String nodesResponseAsString = Strings.toString(serviceContentBuilder); + assertNotNull(nodesResponseAsString); + + final List uploadMetrics = getUploadMetrics(nodeResponse); + for (UploadMetric metric : uploadMetrics) { + XContentBuilder metricContent = XContentFactory.jsonBuilder().startObject(); + metric.toXContent(metricContent, ToXContent.EMPTY_PARAMS); + metricContent.endObject(); + final String metricAsString = Strings.toString(metricContent); + assertNotNull(metricAsString); + assertTrue(nodesResponseAsString.contains(removeStartAndEndObject(metricAsString))); + } + } + + public void testXContentWithTotalUploads() throws IOException { + + Map nodeResponse = randomStatsNodeResponse(); + UploadStatsResponse uploadStatsResponse = new UploadStatsResponse( + new ClusterName(randomLowerCaseString()), + new ArrayList<>(nodeResponse.values()), + emptyList() + ); + final XContentBuilder serviceContentBuilder = jsonBuilder(); + uploadStatsResponse.toXContent(serviceContentBuilder, ToXContent.EMPTY_PARAMS); + String nodesResponseAsString = Strings.toString(serviceContentBuilder); + assertNotNull(nodesResponseAsString); + + TotalUploadStats totalUploadStats = new TotalUploadStats(getUploadStats(nodeResponse)); + XContentBuilder totalUploadStatsContent = XContentFactory.jsonBuilder().startObject(); + totalUploadStats.toXContent(totalUploadStatsContent, ToXContent.EMPTY_PARAMS); + totalUploadStatsContent.endObject(); + final String totalUploadStatsAsString = Strings.toString(totalUploadStatsContent); + assertNotNull(totalUploadStatsAsString); + assertTrue(nodesResponseAsString.contains(removeStartAndEndObject(totalUploadStatsAsString))); + } + + public void testUploadStatsResponseStream() throws IOException { + Map nodeResponse = randomStatsNodeResponse(); + UploadStatsResponse uploadStatsResponse = new UploadStatsResponse( + new ClusterName(randomLowerCaseString()), + new ArrayList<>(nodeResponse.values()), + emptyList() + ); + BytesStreamOutput output = new BytesStreamOutput(); + uploadStatsResponse.writeTo(output); + StreamInput in = StreamInput.wrap(output.bytes().toBytesRef().bytes); + + UploadStatsResponse serializedResponse = new UploadStatsResponse(in); + assertNotNull("serialized response cannot be null", serializedResponse); + } + + private List getUploadStats(Map nodeResponse) { + return nodeResponse.values().stream().map(UploadStatsNodeResponse::getUploadStats).collect(Collectors.toList()); + } + + private List getUploadMetrics(Map nodeResponse) { + return nodeResponse.values() + .stream() + .map(UploadStatsNodeResponse::getUploadStats) + .map(UploadStats::getMetrics) + .flatMap(List::stream) + .collect(Collectors.toList()); + } + +} diff --git a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsServiceTests.java b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsServiceTests.java index d6343e82..b82e6b39 100644 --- a/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/stats/upload/UploadStatsServiceTests.java @@ -7,6 +7,7 @@ import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.geospatial.GeospatialTestHelper.buildFieldNameValuePair; +import static org.opensearch.geospatial.GeospatialTestHelper.removeStartAndEndObject; import java.io.IOException; import java.util.ArrayList; @@ -22,12 +23,6 @@ public class UploadStatsServiceTests extends OpenSearchTestCase { - private String removeStartAndEndObject(String content) { - assertNotNull(content); - assertTrue("content length should be at least 2", content.length() > 1); - return content.substring(1, content.length() - 1); - } - public void testInstanceCreation() { Map randomMap = new HashMap<>(); randomMap.put(GeospatialTestHelper.randomLowerCaseString(), UploadStatsBuilder.randomUploadStats()); @@ -41,24 +36,25 @@ public void testInstanceCreationFails() { assertThrows(NullPointerException.class, () -> new UploadStatsService(null)); } - public void testXContentWithNodeID() { + public void testXContentWithNodeID() throws IOException { Map randomMap = new HashMap<>(); randomMap.put(GeospatialTestHelper.randomLowerCaseString(), UploadStatsBuilder.randomUploadStats()); randomMap.put(GeospatialTestHelper.randomLowerCaseString(), UploadStatsBuilder.randomUploadStats()); UploadStatsService service = new UploadStatsService(randomMap); - String xContent = Strings.toString(service); - assertNotNull(xContent); + final XContentBuilder serviceContentBuilder = jsonBuilder(); + service.toXContent(serviceContentBuilder, ToXContent.EMPTY_PARAMS); + String content = Strings.toString(serviceContentBuilder); + assertNotNull(content); for (String nodeID : randomMap.keySet()) { - assertTrue(nodeID + " is missing", xContent.contains(buildFieldNameValuePair(UploadStatsService.NODE_ID, nodeID))); + assertTrue(nodeID + " is missing", content.contains(buildFieldNameValuePair(UploadStatsService.NODE_ID, nodeID))); } } public void testXContentWithEmptyStats() throws IOException { UploadStatsService service = new UploadStatsService(new HashMap<>()); - final XContentBuilder contentBuilder = jsonBuilder().startObject(); + final XContentBuilder contentBuilder = jsonBuilder(); service.toXContent(contentBuilder, ToXContent.EMPTY_PARAMS); - contentBuilder.endObject(); - String emptyContent = "{\"uploads\":{\"total\":{},\"metrics\":[]}}"; + String emptyContent = "{\"total\":{},\"metrics\":[]}"; assertEquals(emptyContent, Strings.toString(contentBuilder)); } @@ -70,9 +66,8 @@ public void testXContentWithTotalUploadStats() throws IOException { randomMap.put(GeospatialTestHelper.randomLowerCaseString(), stats); } UploadStatsService service = new UploadStatsService(randomMap); - final XContentBuilder serviceContentBuilder = jsonBuilder().startObject(); + final XContentBuilder serviceContentBuilder = jsonBuilder(); service.toXContent(serviceContentBuilder, ToXContent.EMPTY_PARAMS); - serviceContentBuilder.endObject(); String content = Strings.toString(serviceContentBuilder); assertNotNull(content); @@ -94,9 +89,8 @@ public void testXContentWithMetrics() throws IOException { randomMetrics.addAll(stats.getMetrics()); } UploadStatsService service = new UploadStatsService(randomMap); - final XContentBuilder serviceContentBuilder = jsonBuilder().startObject(); + final XContentBuilder serviceContentBuilder = jsonBuilder(); service.toXContent(serviceContentBuilder, ToXContent.EMPTY_PARAMS); - serviceContentBuilder.endObject(); String content = Strings.toString(serviceContentBuilder); assertNotNull(content);