diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java index 4c895b80cfc16..893169ee7f702 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java @@ -24,7 +24,6 @@ import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.blobstore.stream.write.WritePriority; import org.opensearch.common.blobstore.transfer.RemoteTransferContainer; -import org.opensearch.core.compress.Compressor; import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; @@ -35,6 +34,7 @@ import org.opensearch.common.util.io.IOUtils; import org.opensearch.core.action.ActionListener; import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.compress.Compressor; import org.opensearch.core.index.Index; import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.gateway.remote.RemoteStateTransferException; @@ -319,7 +319,11 @@ private void uploadIndexRoutingDiff( ) { BytesReference bytesInput = null; try (BytesStreamOutput streamOutput = new BytesStreamOutput()) { - RemoteIndexRoutingTableDiff remoteIndexRoutingTableDiff = new RemoteIndexRoutingTableDiff(indexRoutingTableDiff, clusterUUID, compressor); + RemoteIndexRoutingTableDiff remoteIndexRoutingTableDiff = new RemoteIndexRoutingTableDiff( + indexRoutingTableDiff, + clusterUUID, + compressor + ); remoteIndexRoutingTableDiff.writeTo(streamOutput); bytesInput = streamOutput.bytes(); } catch (IOException e) { @@ -414,9 +418,10 @@ private RemoteIndexRoutingTable read(BlobContainer blobContainer, String path, I } } - //@Override + // @Override public CheckedRunnable getAsyncIndexRoutingTableDiffReadAction( - String clusterUUID, String uploadedFilename, + String clusterUUID, + String uploadedFilename, LatchedActionListener>> latchedActionListener ) { int idx = uploadedFilename.lastIndexOf("/"); @@ -428,10 +433,7 @@ public CheckedRunnable getAsyncIndexRoutingTableDiffReadAction( clusterUUID, blobContainer, blobFileName, - ActionListener.wrap( - response -> latchedActionListener.onResponse(response.getDiffs()), - latchedActionListener::onFailure - ) + ActionListener.wrap(response -> latchedActionListener.onResponse(response.getDiffs()), latchedActionListener::onFailure) ); } @@ -452,7 +454,10 @@ private RemoteIndexRoutingTableDiff readDiff(String clusterUUID, BlobContainer b try { return new RemoteIndexRoutingTableDiff(blobContainer.readBlob(path), clusterUUID, compressor); } catch (IOException | AssertionError e) { - logger.error(() -> new ParameterizedMessage("RoutingTableDiff read failed for path {}", blobContainer.path().buildAsString() + path), e); + logger.error( + () -> new ParameterizedMessage("RoutingTableDiff read failed for path {}", blobContainer.path().buildAsString() + path), + e + ); throw new RemoteStateTransferException("Failed to read RemoteRoutingTableDiff from Manifest with error ", e); } } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java index 30522b3dd9df0..e418eb5e84182 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/NoopRemoteRoutingTableService.java @@ -39,7 +39,12 @@ public DiffableUtils.MapDiff getIndexRoutingDiffAsyncAction( Map> indexRoutingTableDiff, LatchedActionListener latchedActionListener, BlobPath clusterBasePath - ){ + ) { // noop return () -> {}; } @@ -84,9 +89,9 @@ public CheckedRunnable getAsyncIndexRoutingReadAction( } public CheckedRunnable getAsyncIndexRoutingTableDiffReadAction( - String clusterUUID, - String uploadedFilename, - LatchedActionListener>> latchedActionListener + String clusterUUID, + String uploadedFilename, + LatchedActionListener>> latchedActionListener ) { // noop return () -> {}; diff --git a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java index 8993a545ea880..237c077cb673c 100644 --- a/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java +++ b/server/src/main/java/org/opensearch/common/remote/AbstractRemoteWritableBlobEntity.java @@ -40,10 +40,7 @@ public AbstractRemoteWritableBlobEntity( this.namedXContentRegistry = namedXContentRegistry; } - public AbstractRemoteWritableBlobEntity( - final String clusterUUID, - final Compressor compressor - ) { + public AbstractRemoteWritableBlobEntity(final String clusterUUID, final Compressor compressor) { this(clusterUUID, compressor, null); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 590c0c04549a3..09dff78d13252 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -31,7 +31,6 @@ import static org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer.getAbstractInstance; import static org.opensearch.cluster.DiffableUtils.getStringKeySerializer; -import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.CUSTOM_ROUTING_TABLE_DIFFABLE_VALUE_SERIALIZER; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** @@ -77,7 +76,11 @@ public class ClusterStateDiffManifest implements ToXContentFragment, Writeable { private final List clusterStateCustomUpdated; private final List clusterStateCustomDeleted; - public ClusterStateDiffManifest(ClusterState state, ClusterState previousState, DiffableUtils.MapDiff> routingTableIncrementalDiff) { + public ClusterStateDiffManifest( + ClusterState state, + ClusterState previousState, + DiffableUtils.MapDiff> routingTableIncrementalDiff + ) { fromStateUUID = previousState.stateUUID(); toStateUUID = state.stateUUID(); coordinationMetadataUpdated = !Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 7c55c6d5aafa9..8e01982f35b7a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -428,9 +428,15 @@ public RemoteClusterStateManifestInfo writeIncrementalMetadata( routingTableIncrementalDiff.getDeletes() ); - ClusterStateDiffManifest clusterStateDiffManifest = new ClusterStateDiffManifest(clusterState, previousClusterState, routingTableIncrementalDiff); + ClusterStateDiffManifest clusterStateDiffManifest = new ClusterStateDiffManifest( + clusterState, + previousClusterState, + routingTableIncrementalDiff + ); if (uploadedMetadataResults.uploadedIndicesRoutingDiffMetadata != null) { - clusterStateDiffManifest.setIndicesRoutingDiffPath(uploadedMetadataResults.uploadedIndicesRoutingDiffMetadata.getUploadedFilePath()); + clusterStateDiffManifest.setIndicesRoutingDiffPath( + uploadedMetadataResults.uploadedIndicesRoutingDiffMetadata.getUploadedFilePath() + ); } final RemoteClusterStateManifestInfo manifestDetails = remoteManifestManager.uploadManifest( clusterState, @@ -687,23 +693,13 @@ UploadedMetadataResults writeMetadataInParallel( indicesRoutingToUpload.forEach(indexRoutingTable -> { uploadTasks.put( InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX + indexRoutingTable.getIndex().getName(), - remoteRoutingTableService.getIndexRoutingAsyncAction( - clusterState, - indexRoutingTable, - listener, - clusterBasePath - ) + remoteRoutingTableService.getIndexRoutingAsyncAction(clusterState, indexRoutingTable, listener, clusterBasePath) ); }); if (indexRoutingTableDiff != null && !indexRoutingTableDiff.isEmpty()) { uploadTasks.put( InternalRemoteRoutingTableService.INDEX_ROUTING_DIFF_FILE_PREFIX, - remoteRoutingTableService.getIndexRoutingDiffAsyncAction( - clusterState, - indexRoutingTableDiff, - listener, - clusterBasePath - ) + remoteRoutingTableService.getIndexRoutingDiffAsyncAction(clusterState, indexRoutingTableDiff, listener, clusterBasePath) ); } @@ -1078,7 +1074,9 @@ private ClusterState readClusterStateInParallel( LatchedActionListener>> routingTableDiffLatchedActionListener = new LatchedActionListener<>( ActionListener.wrap(response -> { logger.debug("Successfully read cluster state diff component from remote"); - readIndexRoutingTableDiffResults.set(new RemoteIndexRoutingTableDiff(response, clusterUUID, blobStoreRepository.getCompressor())); + readIndexRoutingTableDiffResults.set( + new RemoteIndexRoutingTableDiff(response, clusterUUID, blobStoreRepository.getCompressor()) + ); }, ex -> { logger.error("Failed to read cluster state diff from remote", ex); exceptionList.add(ex); @@ -1088,9 +1086,11 @@ private ClusterState readClusterStateInParallel( if (readIndexRoutingTableDiff) { asyncMetadataReadActions.add( - remoteRoutingTableService.getAsyncIndexRoutingTableDiffReadAction(clusterUUID, + remoteRoutingTableService.getAsyncIndexRoutingTableDiffReadAction( + clusterUUID, manifest.getDiffManifest().getIndicesRoutingDiffPath(), - routingTableDiffLatchedActionListener) + routingTableDiffLatchedActionListener + ) ); } @@ -1354,7 +1354,10 @@ public ClusterState getClusterStateForManifest( includeEphemeral ? manifest.getIndicesRouting() : emptyList(), includeEphemeral && manifest.getHashesOfConsistentSettings() != null, includeEphemeral ? manifest.getClusterStateCustomMap() : emptyMap(), - includeEphemeral && manifest.getDiffManifest() != null && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), + includeEphemeral + && manifest.getDiffManifest() != null + && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null + && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), includeEphemeral ); } else { @@ -1436,7 +1439,9 @@ public ClusterState getClusterStateUsingDiff( updatedIndexRouting, diff.isHashesOfConsistentSettingsUpdated(), updatedClusterStateCustom, - manifest.getDiffManifest() != null && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), + manifest.getDiffManifest() != null + && manifest.getDiffManifest().getIndicesRoutingDiffPath() != null + && !manifest.getDiffManifest().getIndicesRoutingDiffPath().isEmpty(), true ); ClusterState.Builder clusterStateBuilder = ClusterState.builder(updatedClusterState); diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java index 7bdee7d09da8b..e3c82605847e7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java @@ -101,7 +101,9 @@ RemoteClusterStateManifestInfo uploadManifest( ) { synchronized (this) { if (uploadedMetadataResult.uploadedIndicesRoutingDiffMetadata != null) { - clusterDiffManifest.setIndicesRoutingDiffPath(uploadedMetadataResult.uploadedIndicesRoutingDiffMetadata.getUploadedFilePath()); + clusterDiffManifest.setIndicesRoutingDiffPath( + uploadedMetadataResult.uploadedIndicesRoutingDiffMetadata.getUploadedFilePath() + ); } ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder(); manifestBuilder.clusterTerm(clusterState.term()) diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index cf135225b4fe1..1dc56712d4ab5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -8,9 +8,6 @@ package org.opensearch.gateway.remote.model; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.cluster.routing.remote.InternalRemoteRoutingTableService; import org.opensearch.common.io.Streams; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; diff --git a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiff.java b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiff.java index 1b33f2efa4f0e..9cb4978885a62 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiff.java +++ b/server/src/main/java/org/opensearch/gateway/remote/routingtable/RemoteIndexRoutingTableDiff.java @@ -9,8 +9,8 @@ package org.opensearch.gateway.remote.routingtable; import org.apache.lucene.codecs.CodecUtil; -import org.apache.lucene.store.OutputStreamDataOutput; import org.apache.lucene.store.InputStreamDataInput; +import org.apache.lucene.store.OutputStreamDataOutput; import org.opensearch.cluster.Diff; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; @@ -21,18 +21,25 @@ import org.opensearch.core.common.io.stream.InputStreamStreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.gateway.remote.ClusterMetadataManifest; import org.opensearch.core.compress.Compressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; -import java.util.Map; import java.util.List; +import java.util.Map; -public class RemoteIndexRoutingTableDiff extends AbstractRemoteWritableBlobEntity implements Diff, Writeable { +/** + * Represents a difference between {@link IndexRoutingTable} objects that can be serialized and deserialized. + * This class is responsible for writing and reading the differences between IndexRoutingTables to and from an input/output stream. + */ +public class RemoteIndexRoutingTableDiff extends AbstractRemoteWritableBlobEntity + implements + Diff, + Writeable { private final Map> diffs; diff --git a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java index 839ebe1ff8301..a01672422e03f 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java @@ -250,75 +250,6 @@ public void testGetIndicesRoutingMapDiffIndexAdded() { assertEquals(0, diff.getDeletes().size()); } - public void testGetIndicesRoutingMapDiffShardChanged() { - String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int noOfShards = between(1, 1000); - int noOfReplicas = randomInt(10); - final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") - .build() - ).numberOfShards(noOfShards).numberOfReplicas(noOfReplicas).build(); - - RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); - - final IndexMetadata indexMetadata2 = new IndexMetadata.Builder(indexName).settings( - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") - .build() - ).numberOfShards(noOfShards + 1).numberOfReplicas(noOfReplicas).build(); - RoutingTable routingTable2 = RoutingTable.builder().addAsNew(indexMetadata2).build(); - - DiffableUtils.MapDiff> diff = remoteRoutingTableService - .getIndicesRoutingMapDiff(routingTable, routingTable2); - assertEquals(1, diff.getUpserts().size()); - assertNotNull(diff.getUpserts().get(indexName)); - assertEquals(noOfShards + 1, diff.getUpserts().get(indexName).getShards().size()); - assertEquals(noOfReplicas + 1, diff.getUpserts().get(indexName).getShards().get(0).getSize()); - assertEquals(0, diff.getDeletes().size()); - - final IndexMetadata indexMetadata3 = new IndexMetadata.Builder(indexName).settings( - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") - .build() - ).numberOfShards(noOfShards + 1).numberOfReplicas(noOfReplicas + 1).build(); - RoutingTable routingTable3 = RoutingTable.builder().addAsNew(indexMetadata3).build(); - - diff = remoteRoutingTableService.getIndicesRoutingMapDiff(routingTable2, routingTable3); - assertEquals(1, diff.getUpserts().size()); - assertNotNull(diff.getUpserts().get(indexName)); - assertEquals(noOfShards + 1, diff.getUpserts().get(indexName).getShards().size()); - assertEquals(noOfReplicas + 2, diff.getUpserts().get(indexName).getShards().get(0).getSize()); - - assertEquals(0, diff.getDeletes().size()); - } - - public void testGetIndicesRoutingMapDiffShardDetailChanged() { - String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); - int noOfShards = between(1, 1000); - int noOfReplicas = randomInt(10); - final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetadata.SETTING_INDEX_UUID, "uuid") - .build() - ).numberOfShards(noOfShards).numberOfReplicas(noOfReplicas).build(); - - RoutingTable routingTable = RoutingTable.builder().addAsNew(indexMetadata).build(); - RoutingTable routingTable2 = RoutingTable.builder().addAsRecovery(indexMetadata).build(); - - DiffableUtils.MapDiff> diff = remoteRoutingTableService - .getIndicesRoutingMapDiff(routingTable, routingTable2); - assertEquals(1, diff.getUpserts().size()); - assertNotNull(diff.getUpserts().get(indexName)); - assertEquals(noOfShards, diff.getUpserts().get(indexName).getShards().size()); - assertEquals(noOfReplicas + 1, diff.getUpserts().get(indexName).getShards().get(0).getSize()); - assertEquals(0, diff.getDeletes().size()); - } - public void testGetIndicesRoutingMapDiffIndexDeleted() { String indexName = randomAlphaOfLength(randomIntBetween(1, 50)); final IndexMetadata indexMetadata = new IndexMetadata.Builder(indexName).settings( diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index fd312001b58bd..da6200f1dd5d9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -35,15 +35,11 @@ import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; import static org.opensearch.gateway.remote.model.RemoteCoordinationMetadata.COORDINATION_METADATA; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_METADATA; -import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; import static org.opensearch.gateway.remote.model.RemotePersistentSettingsMetadata.SETTING_METADATA; import static org.opensearch.gateway.remote.model.RemoteTemplatesMetadata.TEMPLATES_METADATA; -import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; public class ClusterMetadataManifestTests extends OpenSearchTestCase { @@ -148,336 +144,6 @@ public void testClusterMetadataManifestXContent() throws IOException { } } - public void testClusterMetadataManifestSerializationEqualsHashCode() { - ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder() - .clusterTerm(1337L) - .stateVersion(7L) - .clusterUUID("HrYF3kP5SmSPWtKlWhnNSA") - .stateUUID("6By9p9G0Rv2MmFYJcPAOgA") - .opensearchVersion(Version.CURRENT) - .nodeId("B10RX1f5RJenMQvYccCgSQ") - .committed(true) - .codecVersion(ClusterMetadataManifest.CODEC_V2) - .indices(randomUploadedIndexMetadataList()) - .previousClusterUUID("yfObdx8KSMKKrXf8UyHhM") - .clusterUUIDCommitted(true) - .coordinationMetadata(new UploadedMetadataAttribute(COORDINATION_METADATA, "coordination-file")) - .settingMetadata(new UploadedMetadataAttribute(SETTING_METADATA, "setting-file")) - .templatesMetadata(new UploadedMetadataAttribute(TEMPLATES_METADATA, "templates-file")) - .customMetadataMap( - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - CUSTOM_METADATA + CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - CUSTOM_METADATA + CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - CUSTOM_METADATA + CUSTOM_DELIMITER + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" - ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) - ) - .routingTableVersion(1L) - .discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, "discovery-nodes-file")) - .clusterBlocksMetadata(new UploadedMetadataAttribute(CLUSTER_BLOCKS, "cluster-block-file")) - .transientSettingsMetadata(new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, "transient-settings-file")) - .hashesOfConsistentSettings(new UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, "hashes-of-consistent-settings-file")) - .clusterStateCustomMetadataMap(Collections.emptyMap()) - .diffManifest( - new ClusterStateDiffManifest( - RemoteClusterStateServiceTests.generateClusterStateWithOneIndex().build(), - ClusterState.EMPTY_STATE, - null - ) - ) - .build(); - { // Mutate Cluster Term - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.clusterTerm(1338L); - return builder.build(); - } - ); - } - { // Mutate State Version - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.stateVersion(8L); - return builder.build(); - } - ); - } - { // Mutate Cluster UUID - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.clusterUUID("efOkMiPbQZCUQQgtFWdbPw"); - return builder.build(); - } - ); - } - { // Mutate State UUID - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.stateUUID("efOkMiPbQZCUQQgtFWdbPw"); - return builder.build(); - } - ); - } - { // Mutate OpenSearch Version - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.opensearchVersion(Version.V_EMPTY); - return builder.build(); - } - ); - } - { // Mutate Committed State - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.committed(false); - return builder.build(); - } - ); - } - { // Mutate Indices - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.indices(randomUploadedIndexMetadataList()); - return builder.build(); - } - ); - } - { // Mutate Previous cluster UUID - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.previousClusterUUID("vZX62DCQEOzGXlxXCrEu"); - return builder.build(); - } - ); - - } - { // Mutate cluster uuid committed - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.clusterUUIDCommitted(false); - return builder.build(); - } - ); - } - { - // Mutate Coordination metadata - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.coordinationMetadata(randomUploadedMetadataAttribute()); - return builder.build(); - } - ); - } - { - // Mutate setting metadata - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.settingMetadata(randomUploadedMetadataAttribute()); - return builder.build(); - } - ); - } - { - // Mutate template metadata - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.templatesMetadata(randomUploadedMetadataAttribute()); - return builder.build(); - } - ); - } - { - // Mutate custom metadata - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.customMetadataMap(Collections.emptyMap()); - return builder.build(); - } - ); - } - { - // Mutate discovery nodes - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.discoveryNodesMetadata(randomUploadedMetadataAttribute()); - return builder.build(); - } - ); - } - { - // Mutate cluster blocks - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.clusterBlocksMetadata(randomUploadedMetadataAttribute()); - return builder.build(); - } - ); - } - { - // Mutate transient settings metadata - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.transientSettingsMetadata(randomUploadedMetadataAttribute()); - return builder.build(); - } - ); - } - { - // Mutate diff manifest - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.diffManifest(null); - return builder.build(); - } - ); - } - { - // Mutate hashes of consistent settings - EqualsHashCodeTestUtils.checkEqualsAndHashCode( - initialManifest, - orig -> OpenSearchTestCase.copyWriteable( - orig, - new NamedWriteableRegistry(Collections.emptyList()), - ClusterMetadataManifest::new - ), - manifest -> { - ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); - builder.hashesOfConsistentSettings(randomUploadedMetadataAttribute()); - return builder.build(); - } - ); - } - } - public void testClusterMetadataManifestXContentV2() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 53cf66b128b83..860aa78292f20 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -93,7 +93,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatchers; -import org.mockito.Mockito; import static java.util.stream.Collectors.toList; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; @@ -113,7 +112,6 @@ import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; @@ -121,7 +119,6 @@ import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -511,155 +508,6 @@ public void testFailWriteIncrementalMetadataWhenTermChanged() { ); } - public void testWriteIncrementalMetadataSuccess() throws IOException { - final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); - mockBlobStoreObjects(); - final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); - final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) - .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) - .build(); - - final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(Collections.emptyList()).build(); - - remoteClusterStateService.start(); - final RemoteClusterStateService rcssSpy = Mockito.spy(remoteClusterStateService); - final RemoteClusterStateManifestInfo manifestInfo = rcssSpy.writeIncrementalMetadata( - previousClusterState, - clusterState, - previousManifest - ); - final ClusterMetadataManifest manifest = manifestInfo.getClusterMetadataManifest(); - final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename__2"); - final List indices = List.of(uploadedIndexMetadata); - - final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() - .indices(indices) - .clusterTerm(1L) - .stateVersion(1L) - .stateUUID("state-uuid") - .clusterUUID("cluster-uuid") - .previousClusterUUID("prev-cluster-uuid") - .build(); - - Mockito.verify(rcssSpy) - .writeMetadataInParallel( - eq(clusterState), - eq(new ArrayList(clusterState.metadata().indices().values())), - eq(Collections.singletonMap(indices.get(0).getIndexName(), null)), - eq(clusterState.metadata().customs()), - eq(true), - eq(true), - eq(true), - eq(false), - eq(false), - eq(false), - eq(Collections.emptyMap()), - eq(false), - eq(Collections.emptyList()), - eq(null) - ); - - assertThat(manifestInfo.getManifestFileName(), notNullValue()); - assertThat(manifest.getIndices().size(), is(1)); - assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName())); - assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID())); - assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue()); - assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); - assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); - assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); - assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); - assertThat(manifest.getHashesOfConsistentSettings(), nullValue()); - assertThat(manifest.getDiscoveryNodesMetadata(), nullValue()); - assertThat(manifest.getClusterBlocksMetadata(), nullValue()); - assertThat(manifest.getClusterStateCustomMap(), anEmptyMap()); - assertThat(manifest.getTransientSettingsMetadata(), nullValue()); - assertThat(manifest.getTemplatesMetadata(), notNullValue()); - assertThat(manifest.getCoordinationMetadata(), notNullValue()); - assertThat(manifest.getCustomMetadataMap().size(), is(2)); - assertThat(manifest.getIndicesRouting().size(), is(0)); - } - - public void testWriteIncrementalMetadataSuccessWhenPublicationEnabled() throws IOException { - publicationEnabled = true; - Settings nodeSettings = Settings.builder().put(REMOTE_PUBLICATION_EXPERIMENTAL, publicationEnabled).build(); - FeatureFlags.initializeFeatureFlags(nodeSettings); - remoteClusterStateService = new RemoteClusterStateService( - "test-node-id", - repositoriesServiceSupplier, - settings, - clusterService, - () -> 0L, - threadPool, - List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings)), - writableRegistry() - ); - final ClusterState clusterState = generateClusterStateWithOneIndex().nodes(nodesWithLocalNodeClusterManager()).build(); - mockBlobStoreObjects(); - final CoordinationMetadata coordinationMetadata = CoordinationMetadata.builder().term(1L).build(); - final ClusterState previousClusterState = ClusterState.builder(ClusterName.DEFAULT) - .metadata(Metadata.builder().coordinationMetadata(coordinationMetadata)) - .build(); - - final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder().indices(Collections.emptyList()).build(); - - remoteClusterStateService.start(); - final RemoteClusterStateService rcssSpy = Mockito.spy(remoteClusterStateService); - final RemoteClusterStateManifestInfo manifestInfo = rcssSpy.writeIncrementalMetadata( - previousClusterState, - clusterState, - previousManifest - ); - final ClusterMetadataManifest manifest = manifestInfo.getClusterMetadataManifest(); - final UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "index-uuid", "metadata-filename__2"); - final List indices = List.of(uploadedIndexMetadata); - - final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() - .indices(indices) - .clusterTerm(1L) - .stateVersion(1L) - .stateUUID("state-uuid") - .clusterUUID("cluster-uuid") - .previousClusterUUID("prev-cluster-uuid") - .build(); - - Mockito.verify(rcssSpy) - .writeMetadataInParallel( - eq(clusterState), - eq(new ArrayList(clusterState.metadata().indices().values())), - eq(Collections.singletonMap(indices.get(0).getIndexName(), null)), - eq(clusterState.metadata().customs()), - eq(true), - eq(true), - eq(true), - eq(true), - eq(false), - eq(false), - eq(Collections.emptyMap()), - eq(true), - Mockito.anyList(), - eq(null) - ); - - assertThat(manifestInfo.getManifestFileName(), notNullValue()); - assertThat(manifest.getIndices().size(), is(1)); - assertThat(manifest.getIndices().get(0).getIndexName(), is(uploadedIndexMetadata.getIndexName())); - assertThat(manifest.getIndices().get(0).getIndexUUID(), is(uploadedIndexMetadata.getIndexUUID())); - assertThat(manifest.getIndices().get(0).getUploadedFilename(), notNullValue()); - assertThat(manifest.getClusterTerm(), is(expectedManifest.getClusterTerm())); - assertThat(manifest.getStateVersion(), is(expectedManifest.getStateVersion())); - assertThat(manifest.getClusterUUID(), is(expectedManifest.getClusterUUID())); - assertThat(manifest.getStateUUID(), is(expectedManifest.getStateUUID())); - assertThat(manifest.getHashesOfConsistentSettings(), notNullValue()); - assertThat(manifest.getDiscoveryNodesMetadata(), notNullValue()); - assertThat(manifest.getClusterBlocksMetadata(), nullValue()); - assertThat(manifest.getClusterStateCustomMap(), anEmptyMap()); - assertThat(manifest.getTransientSettingsMetadata(), nullValue()); - assertThat(manifest.getTemplatesMetadata(), notNullValue()); - assertThat(manifest.getCoordinationMetadata(), notNullValue()); - assertThat(manifest.getCustomMetadataMap().size(), is(2)); - assertThat(manifest.getIndicesRouting().size(), is(1)); - } - /* * Here we will verify the migration of manifest file from codec V0. *