From 255075c0dabafb598adc50a9b1a4cbd16b212b12 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Wed, 5 Jun 2024 20:55:10 +0530 Subject: [PATCH 01/12] Add POJO classes required for cluster state publication from remote Signed-off-by: Shivansh Arora --- .../opensearch/cluster/metadata/Metadata.java | 4 + .../remote/ClusterMetadataManifest.java | 292 +++++++++- .../remote/ClusterStateDiffManifest.java | 549 ++++++++++++++++++ .../RemoteClusterStateAttributesManager.java | 134 +++++ .../remote/model/RemoteClusterBlocks.java | 102 ++++ .../model/RemoteClusterStateCustoms.java | 108 ++++ .../remote/model/RemoteDiscoveryNodes.java | 101 ++++ .../RemoteHashesOfConsistentSettings.java | 94 +++ .../remote/model/RemoteReadResult.java | 39 ++ .../RemoteTransientSettingsMetadata.java | 101 ++++ 10 files changed, 1505 insertions(+), 19 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java create mode 100644 server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index e1aa5626f36c1..232f900f25375 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -973,6 +973,10 @@ public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metad return metadata1.persistentSettings.equals(metadata2.persistentSettings); } + public static boolean isTransientSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.transientSettings.equals(metadata2.transientSettings); + } + public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) { return metadata1.templates.equals(metadata2.templates); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index b3b1bf37f8696..e2963ee228175 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -8,6 +8,8 @@ package org.opensearch.gateway.remote; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.core.ParseField; import org.opensearch.core.common.Strings; @@ -41,7 +43,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. - public static final int CODEC_V3 = 3; // In Codec V3, we introduce index routing-metadata in manifest file. + public static final int CODEC_V3 = 3; // In Codec V3, we introduce index routing-metadata, diff and other attributes as part of manifest required for state publication private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -61,6 +63,13 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata"); private static final ParseField ROUTING_TABLE_VERSION_FIELD = new ParseField("routing_table_version"); private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); + private static final ParseField METADATA_VERSION = new ParseField("metadata_version"); + private static final ParseField UPLOADED_TRANSIENT_SETTINGS_METADATA = new ParseField("uploaded_transient_settings_metadata"); + private static final ParseField UPLOADED_DISCOVERY_NODES_METADATA = new ParseField("uploaded_discovery_nodes_metadata"); + private static final ParseField UPLOADED_CLUSTER_BLOCKS_METADATA = new ParseField("uploaded_cluster_blocks_metadata"); + private static final ParseField UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA = new ParseField("uploaded_hashes_of_consistent_settings_metadata"); + private static final ParseField UPLOADED_CLUSTER_STATE_CUSTOM_METADATA = new ParseField("uploaded_cluster_state_custom_metadata"); + private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest"); private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields) { return ClusterMetadataManifest.builder() @@ -90,9 +99,16 @@ private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields } private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields) { - return manifestV2Builder(fields).codecVersion(codecVersion(fields)) + return manifestV2Builder(fields) .routingTableVersion(routingTableVersion(fields)) - .indicesRouting(indicesRouting(fields)); + .indicesRouting(indicesRouting(fields)) + .discoveryNodesMetadata(discoveryNodesMetadata(fields)) + .clusterBlocksMetadata(clusterBlocksMetadata(fields)) + .diffManifest(diffManifest(fields)) + .metadataVersion(metadataVersion(fields)) + .transientSettingsMetadata(transientSettingsMetadata(fields)) + .hashesOfConsistentSettings(hashesOfConsistentSettings(fields)) + .clusterStateCustomMetadataMap(clusterStateCustomMetadata(fields)); } private static long term(Object[] fields) { @@ -168,6 +184,35 @@ private static List indicesRouting(Object[] fields) { return (List) fields[16]; } + private static UploadedMetadataAttribute discoveryNodesMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[17]; + } + + private static UploadedMetadataAttribute clusterBlocksMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[18]; + } + + private static long metadataVersion(Object[] fields) { + return (long) fields[19]; + } + + private static UploadedMetadataAttribute transientSettingsMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[20]; + } + + private static UploadedMetadataAttribute hashesOfConsistentSettings(Object[] fields) { + return (UploadedMetadataAttribute) fields[21]; + } + + private static Map clusterStateCustomMetadata(Object[] fields) { + List customs = (List) fields[22]; + return customs.stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())); + } + + private static ClusterStateDiffManifest diffManifest(Object[] fields) { + return (ClusterStateDiffManifest) fields[23]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> manifestV0Builder(fields).build() @@ -246,6 +291,37 @@ private static void declareParser(ConstructingObjectParser UploadedIndexMetadata.fromXContent(p), INDICES_ROUTING_FIELD ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_DISCOVERY_NODES_METADATA + ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_CLUSTER_BLOCKS_METADATA + ); + parser.declareLong(ConstructingObjectParser.constructorArg(), METADATA_VERSION); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_TRANSIENT_SETTINGS_METADATA + ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA + ); + parser.declareNamedObjects( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_CLUSTER_STATE_CUSTOM_METADATA + ); + parser.declareObject( + ConstructingObjectParser.constructorArg(), + (p, c) -> ClusterStateDiffManifest.fromXContent(p), + DIFF_MANIFEST + ); } } @@ -267,6 +343,13 @@ private static void declareParser(ConstructingObjectParser indicesRouting; + private final long metadataVersion; + private final UploadedMetadataAttribute uploadedTransientSettingsMetadata; + private final UploadedMetadataAttribute uploadedDiscoveryNodesMetadata; + private final UploadedMetadataAttribute uploadedClusterBlocksMetadata; + private final UploadedMetadataAttribute uploadedHashesOfConsistentSettings; + private final Map uploadedClusterStateCustomMap; + private final ClusterStateDiffManifest diffManifest; public List getIndices() { return indices; @@ -332,6 +415,38 @@ public Map getCustomMetadataMap() { return uploadedCustomMetadataMap; } + public long getMetadataVersion() { + return metadataVersion; + } + + public UploadedMetadataAttribute getTransientSettingsMetadata() { + return uploadedTransientSettingsMetadata; + } + + public UploadedMetadataAttribute getDiscoveryNodesMetadata() { + return uploadedDiscoveryNodesMetadata; + } + + public UploadedMetadataAttribute getClusterBlocksMetadata() { + return uploadedClusterBlocksMetadata; + } + + public ClusterStateDiffManifest getDiffManifest() { + return diffManifest; + } + + public UploadedMetadataAttribute getDiscoverNodeMetadata() { + return uploadedDiscoveryNodesMetadata; + } + + public Map getClusterStateCustomMap() { + return uploadedClusterStateCustomMap; + } + + public UploadedMetadataAttribute getHashesOfConsistentSettings() { + return uploadedHashesOfConsistentSettings; + } + public boolean hasMetadataAttributesFiles() { return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null @@ -365,7 +480,14 @@ public ClusterMetadataManifest( UploadedMetadataAttribute uploadedTemplatesMetadata, Map uploadedCustomMetadataMap, long routingTableVersion, - List indicesRouting + List indicesRouting, + long metadataVersion, + UploadedMetadataAttribute discoveryNodesMetadata, + UploadedMetadataAttribute clusterBlocksMetadata, + UploadedMetadataAttribute uploadedTransientSettingsMetadata, + UploadedMetadataAttribute uploadedHashesOfConsistentSettings, + Map uploadedClusterStateCustomMap, + ClusterStateDiffManifest diffManifest ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -387,6 +509,15 @@ public ClusterMetadataManifest( this.uploadedCustomMetadataMap = Collections.unmodifiableMap( uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>() ); + this.uploadedDiscoveryNodesMetadata = discoveryNodesMetadata; + this.uploadedClusterBlocksMetadata = clusterBlocksMetadata; + this.diffManifest = diffManifest; + this.metadataVersion = metadataVersion; + this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata; + this.uploadedHashesOfConsistentSettings = uploadedHashesOfConsistentSettings; + this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( + uploadedClusterStateCustomMap != null ? uploadedClusterStateCustomMap : new HashMap<>() + ); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -411,25 +542,36 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.globalMetadataFileName = null; this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); - } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { - this.codecVersion = in.readInt(); - this.globalMetadataFileName = in.readString(); - this.uploadedCoordinationMetadata = null; - this.uploadedSettingsMetadata = null; - this.uploadedTemplatesMetadata = null; - this.uploadedCustomMetadataMap = null; - this.routingTableVersion = -1; - this.indicesRouting = null; + this.metadataVersion = in.readLong(); + this.uploadedDiscoveryNodesMetadata = new UploadedMetadataAttribute(in); + this.uploadedClusterBlocksMetadata = new UploadedMetadataAttribute(in); + this.uploadedTransientSettingsMetadata = new UploadedMetadataAttribute(in); + this.uploadedHashesOfConsistentSettings = new UploadedMetadataAttribute(in); + this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( + in.readMap(StreamInput::readString, UploadedMetadataAttribute::new) + ); + this.diffManifest = null; } else { - this.codecVersion = CODEC_V0; // Default codec - this.globalMetadataFileName = null; + if (in.getVersion().onOrAfter(Version.V_2_12_0)) { + this.codecVersion = in.readInt(); + this.globalMetadataFileName = in.readString(); + } else { + this.codecVersion = CODEC_V0; // Default codec + this.globalMetadataFileName = null; + } this.uploadedCoordinationMetadata = null; this.uploadedSettingsMetadata = null; this.uploadedTemplatesMetadata = null; this.uploadedCustomMetadataMap = null; this.routingTableVersion = -1; this.indicesRouting = null; - } + this.uploadedDiscoveryNodesMetadata = null; + this.uploadedClusterBlocksMetadata = null; + this.diffManifest = null; + this.metadataVersion = -1; + this.uploadedTransientSettingsMetadata = null; + this.uploadedHashesOfConsistentSettings = null; + this.uploadedClusterStateCustomMap = null; } public static Builder builder() { @@ -497,6 +639,37 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endArray(); + if (getDiscoveryNodesMetadata() != null) { + builder.startObject(UPLOADED_DISCOVERY_NODES_METADATA.getPreferredName()); + getDiscoveryNodesMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getClusterBlocksMetadata() != null) { + builder.startObject(UPLOADED_CLUSTER_BLOCKS_METADATA.getPreferredName()); + getClusterBlocksMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getTransientSettingsMetadata() != null) { + builder.startObject(UPLOADED_TRANSIENT_SETTINGS_METADATA.getPreferredName()); + getTransientSettingsMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getDiffManifest() != null) { + builder.startObject(DIFF_MANIFEST.getPreferredName()); + getDiffManifest().toXContent(builder, params); + builder.endObject(); + } + builder.field(METADATA_VERSION.getPreferredName(), getMetadataVersion()); + if (getHashesOfConsistentSettings() != null) { + builder.startObject(UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA.getPreferredName()); + getHashesOfConsistentSettings().toXContent(builder, params); + builder.endObject(); + } + builder.startObject(UPLOADED_CLUSTER_STATE_CUSTOM_METADATA.getPreferredName()); + for (UploadedMetadataAttribute attribute : getClusterStateCustomMap().values()) { + attribute.toXContent(builder, params); + } + builder.endObject(); } return builder; } @@ -521,6 +694,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(uploadedCustomMetadataMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); + out.writeLong(metadataVersion); + uploadedDiscoveryNodesMetadata.writeTo(out); + uploadedClusterBlocksMetadata.writeTo(out); + uploadedTransientSettingsMetadata.writeTo(out); + uploadedHashesOfConsistentSettings.writeTo(out); + out.writeMap(uploadedClusterStateCustomMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); @@ -529,6 +708,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public boolean equals(Object o) { + // ToDo: update this method to contain new attributes if (this == o) { return true; } @@ -549,11 +729,22 @@ public boolean equals(Object o) { && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) && Objects.equals(codecVersion, that.codecVersion) && Objects.equals(routingTableVersion, that.routingTableVersion) - && Objects.equals(indicesRouting, that.indicesRouting); + && Objects.equals(indicesRouting, that.indicesRouting) + && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) + && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) + && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) + && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) + && Objects.equals(metadataVersion, that.metadataVersion) + && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) + && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) + && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) + && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) + && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); } @Override public int hashCode() { + // ToDo: update this method to contain new attributes return Objects.hash( codecVersion, globalMetadataFileName, @@ -568,7 +759,17 @@ public int hashCode() { previousClusterUUID, clusterUUIDCommitted, routingTableVersion, - indicesRouting + indicesRouting, + uploadedCoordinationMetadata, + uploadedSettingsMetadata, + uploadedTemplatesMetadata, + uploadedCustomMetadataMap, + metadataVersion, + uploadedDiscoveryNodesMetadata, + uploadedClusterBlocksMetadata, + uploadedTransientSettingsMetadata, + uploadedHashesOfConsistentSettings, + uploadedClusterStateCustomMap ); } @@ -622,6 +823,13 @@ public static class Builder { private boolean clusterUUIDCommitted; private long routingTableVersion; private List indicesRouting; + private long metadataVersion; + private UploadedMetadataAttribute discoveryNodesMetadata; + private UploadedMetadataAttribute clusterBlocksMetadata; + private UploadedMetadataAttribute transientSettingsMetadata; + private UploadedMetadataAttribute hashesOfConsistentSettings; + private Map clusterStateCustomMetadataMap; + private ClusterStateDiffManifest diffManifest; public Builder indices(List indices) { this.indices = indices; @@ -726,10 +934,46 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { return this; } + public Builder metadataVersion(long metadataVersion) { + this.metadataVersion = metadataVersion; + return this; + } + + public Builder discoveryNodesMetadata(UploadedMetadataAttribute discoveryNodesMetadata) { + this.discoveryNodesMetadata = discoveryNodesMetadata; + return this; + } + + public Builder clusterBlocksMetadata(UploadedMetadataAttribute clusterBlocksMetadata) { + this.clusterBlocksMetadata = clusterBlocksMetadata; + return this; + } + + public Builder transientSettingsMetadata(UploadedMetadataAttribute settingsMetadata) { + this.transientSettingsMetadata = settingsMetadata; + return this; + } + + public Builder hashesOfConsistentSettings(UploadedMetadataAttribute hashesOfConsistentSettings) { + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + return this; + } + + public Builder clusterStateCustomMetadataMap(Map clusterStateCustomMetadataMap) { + this.clusterStateCustomMetadataMap = clusterStateCustomMetadataMap; + return this; + } + + public Builder diffManifest(ClusterStateDiffManifest diffManifest) { + this.diffManifest = diffManifest; + return this; + } + public Builder() { indices = new ArrayList<>(); customMetadataMap = new HashMap<>(); indicesRouting = new ArrayList<>(); + clusterStateCustomMetadataMap = new HashMap<>(); } public Builder(ClusterMetadataManifest manifest) { @@ -751,6 +995,9 @@ public Builder(ClusterMetadataManifest manifest) { this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; this.routingTableVersion = manifest.routingTableVersion; this.indicesRouting = new ArrayList<>(manifest.indicesRouting); + this.diffManifest = manifest.diffManifest; + this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings; + this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap; } public ClusterMetadataManifest build() { @@ -772,7 +1019,14 @@ public ClusterMetadataManifest build() { templatesMetadata, customMetadataMap, routingTableVersion, - indicesRouting + indicesRouting, + metadataVersion, + discoveryNodesMetadata, + clusterBlocksMetadata, + transientSettingsMetadata, + hashesOfConsistentSettings, + clusterStateCustomMetadataMap, + diffManifest ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java new file mode 100644 index 0000000000000..ea4344d4d35e8 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -0,0 +1,549 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.core.common.Strings; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; + +public class ClusterStateDiffManifest implements ToXContentObject { + private static final String FROM_STATE_UUID_FIELD = "from_state_uuid"; + private static final String TO_STATE_UUID_FIELD = "to_state_uuid"; + private static final String METADATA_DIFF_FIELD = "metadata_diff"; + private static final String COORDINATION_METADATA_UPDATED_FIELD = "coordination_metadata_diff"; + private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff"; + private static final String TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD = "transient_settings_metadata_diff"; + private static final String TEMPLATES_METADATA_UPDATED_FIELD = "templates_metadata_diff"; + private static final String HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD = "hashes_of_consistent_settings_diff"; + private static final String INDICES_DIFF_FIELD = "indices_diff"; + private static final String METADATA_CUSTOM_DIFF_FIELD = "metadata_custom_diff"; + private static final String UPSERTS_FIELD = "upserts"; + private static final String DELETES_FIELD = "deletes"; + private static final String CLUSTER_BLOCKS_UPDATED_FIELD = "cluster_blocks_diff"; + private static final String DISCOVERY_NODES_UPDATED_FIELD = "discovery_nodes_diff"; + private static final String CLUSTER_STATE_CUSTOM_DIFF_FIELD = "cluster_state_custom_diff"; + + private final String fromStateUUID; + private final String toStateUUID; + private final boolean coordinationMetadataUpdated; + private final boolean settingsMetadataUpdated; + private final boolean transientSettingsMetadataUpdate; + private final boolean templatesMetadataUpdated; + private List customMetadataUpdated; + private final List customMetadataDeleted; + private final List indicesUpdated; + private final List indicesDeleted; + private final boolean clusterBlocksUpdated; + private final boolean discoveryNodesUpdated; + private final boolean hashesOfConsistentSettingsUpdated; + private final List clusterStateCustomUpdated; + private final List clusterStateCustomDeleted; + + ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { + fromStateUUID = previousState.stateUUID(); + toStateUUID = state.stateUUID(); + coordinationMetadataUpdated = !Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); + settingsMetadataUpdated = !Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); + transientSettingsMetadataUpdate = !Metadata.isTransientSettingsMetadataEqual(state.metadata(), previousState.metadata()); + templatesMetadataUpdated = !Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); + indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); + indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); + clusterBlocksUpdated = !state.blocks().equals(previousState.blocks()); + discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); + customMetadataUpdated = new ArrayList<>(); + for (String custom : state.metadata().customs().keySet()) { + if (!previousState.metadata().customs().containsKey(custom) || !state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) { + customMetadataUpdated.add(custom); + } + } + customMetadataDeleted = new ArrayList<>(); + for (String custom : previousState.metadata().customs().keySet()) { + if (state.metadata().customs().get(custom) == null) { + customMetadataDeleted.add(custom); + } + } + + hashesOfConsistentSettingsUpdated = !state.metadata().hashesOfConsistentSettings().equals(previousState.metadata().hashesOfConsistentSettings()); + clusterStateCustomUpdated = new ArrayList<>(); + clusterStateCustomDeleted = new ArrayList<>(); + for (String custom : state.customs().keySet()) { + if (!previousState.customs().containsKey(custom) || !state.customs().get(custom).equals(previousState.customs().get(custom))) { + clusterStateCustomUpdated.add(custom); + } + } + for (String custom : previousState.customs().keySet()) { + if (state.customs().get(custom) == null) { + clusterStateCustomDeleted.add(custom); + } + } + } + + public ClusterStateDiffManifest( + String fromStateUUID, + String toStateUUID, + boolean coordinationMetadataUpdated, + boolean settingsMetadataUpdated, + boolean transientSettingsMetadataUpdate, + boolean templatesMetadataUpdated, + List customMetadataUpdated, + List customMetadataDeleted, + List indicesUpdated, + List indicesDeleted, + boolean clusterBlocksUpdated, + boolean discoveryNodesUpdated, + boolean hashesOfConsistentSettingsUpdated, + List clusterStateCustomUpdated, + List clusterStateCustomDeleted + ) { + this.fromStateUUID = fromStateUUID; + this.toStateUUID = toStateUUID; + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + this.settingsMetadataUpdated = settingsMetadataUpdated; + this.transientSettingsMetadataUpdate = transientSettingsMetadataUpdate; + this.templatesMetadataUpdated = templatesMetadataUpdated; + this.customMetadataUpdated = customMetadataUpdated; + this.customMetadataDeleted = customMetadataDeleted; + this.indicesUpdated = indicesUpdated; + this.indicesDeleted = indicesDeleted; + this.clusterBlocksUpdated = clusterBlocksUpdated; + this.discoveryNodesUpdated = discoveryNodesUpdated; + this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated; + this.clusterStateCustomUpdated = clusterStateCustomUpdated; + this.clusterStateCustomDeleted = clusterStateCustomDeleted; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + { + builder.field(FROM_STATE_UUID_FIELD, fromStateUUID); + builder.field(TO_STATE_UUID_FIELD, toStateUUID); + builder.startObject(METADATA_DIFF_FIELD); + { + builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); + builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); + builder.field(TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD, transientSettingsMetadataUpdate); + builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); + builder.startObject(INDICES_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String index : indicesUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String index : indicesDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + builder.startObject(METADATA_CUSTOM_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String custom : customMetadataUpdated) { + builder.value(custom); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String custom : customMetadataDeleted) { + builder.value(custom); + } + builder.endArray(); + builder.endObject(); + builder.field(HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD, hashesOfConsistentSettingsUpdated); + } + builder.endObject(); + builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); + builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); + builder.startObject(CLUSTER_STATE_CUSTOM_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String custom : clusterStateCustomUpdated) { + builder.value(custom); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String custom : clusterStateCustomDeleted) { + builder.value(custom); + } + builder.endArray(); + builder.endObject(); + + } + return builder; + } + + public static ClusterStateDiffManifest fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(METADATA_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token.isValue()) { + switch (currentFieldName) { + case COORDINATION_METADATA_UPDATED_FIELD: + builder.coordinationMetadataUpdated(parser.booleanValue()); + break; + case SETTINGS_METADATA_UPDATED_FIELD: + builder.settingsMetadataUpdated(parser.booleanValue()); + break; + case TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD: + builder.transientSettingsMetadataUpdate(parser.booleanValue()); + break; + case TEMPLATES_METADATA_UPDATED_FIELD: + builder.templatesMetadataUpdated(parser.booleanValue()); + break; + case HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD: + builder.hashesOfConsistentSettingsUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(INDICES_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.indicesUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.indicesDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else if (currentFieldName.equals(METADATA_CUSTOM_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.customMetadataUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.customMetadataDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + } else if (currentFieldName.equals(CLUSTER_STATE_CUSTOM_DIFF_FIELD)) { + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.clusterStateCustomUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.clusterStateCustomDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token.isValue()) { + switch (currentFieldName) { + case FROM_STATE_UUID_FIELD: + builder.fromStateUUID(parser.text()); + break; + case TO_STATE_UUID_FIELD: + builder.toStateUUID(parser.text()); + break; + case CLUSTER_BLOCKS_UPDATED_FIELD: + builder.clusterBlocksUpdated(parser.booleanValue()); + break; + case DISCOVERY_NODES_UPDATED_FIELD: + builder.discoveryNodesUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + return builder.build(); + } + + @Override + public String toString() { + return Strings.toString(MediaTypeRegistry.JSON, this); + } + + public List findRemovedIndices(Map indices, Map previousIndices) { + List removedIndices = new ArrayList<>(); + for (String index : previousIndices.keySet()) { + // index present in previous state but not in current + if (!indices.containsKey(index)) { + removedIndices.add(index); + } + } + return removedIndices; + } + + public List findUpdatedIndices(Map indices, Map previousIndices) { + List updatedIndices = new ArrayList<>(); + for (String index : indices.keySet()) { + if (!previousIndices.containsKey(index)) { + updatedIndices.add(index); + } else if (previousIndices.get(index).getVersion() != indices.get(index).getVersion()) { + updatedIndices.add(index); + } + } + return updatedIndices; + } + + public List getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List deletedIndicesRouting = new ArrayList<>(); + for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { + if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is deleted + deletedIndicesRouting.add(previousIndexRouting.getIndex().getName()); + } + } + return deletedIndicesRouting; + } + + public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { + List updatedIndicesRouting = new ArrayList<>(); + for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { + if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { + // Latest Routing Table does not have entry for the index which means the index is created + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } else { + if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { + // if the latest routing table has the same routing table as the previous routing table, then the index is not updated + continue; + } + updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); + } + } + return updatedIndicesRouting; + } + + public String getFromStateUUID() { + return fromStateUUID; + } + + public String getToStateUUID() { + return toStateUUID; + } + + public boolean isCoordinationMetadataUpdated() { + return coordinationMetadataUpdated; + } + + public boolean isSettingsMetadataUpdated() { + return settingsMetadataUpdated; + } + + public boolean isTransientSettingsMetadataUpdated() { + return transientSettingsMetadataUpdate; + } + + public boolean isTemplatesMetadataUpdated() { + return templatesMetadataUpdated; + } + + public List getCustomMetadataUpdated() { + return customMetadataUpdated; + } + + public List getCustomMetadataDeleted() { + return customMetadataDeleted; + } + + public List getIndicesUpdated() { + return indicesUpdated; + } + + public List getIndicesDeleted() { + return indicesDeleted; + } + + public boolean isClusterBlocksUpdated() { + return clusterBlocksUpdated; + } + + public boolean isDiscoveryNodesUpdated() { + return discoveryNodesUpdated; + } + + public boolean isHashesOfConsistentSettingsUpdated() { + return hashesOfConsistentSettingsUpdated; + } + + public List getClusterStateCustomUpdated() { + return clusterStateCustomUpdated; + } + + public List getClusterStateCustomDeleted() { + return clusterStateCustomDeleted; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String fromStateUUID; + private String toStateUUID; + private boolean coordinationMetadataUpdated; + private boolean settingsMetadataUpdated; + private boolean transientSettingsMetadataUpdated; + private boolean templatesMetadataUpdated; + private List customMetadataUpdated; + private List customMetadataDeleted; + private List indicesUpdated; + private List indicesDeleted; + private boolean clusterBlocksUpdated; + private boolean discoveryNodesUpdated; + private boolean hashesOfConsistentSettingsUpdated; + private List clusterStateCustomUpdated; + private List clusterStateCustomDeleted; + + public Builder() {} + + public Builder fromStateUUID(String fromStateUUID) { + this.fromStateUUID = fromStateUUID; + return this; + } + + public Builder toStateUUID(String toStateUUID) { + this.toStateUUID = toStateUUID; + return this; + } + + public Builder coordinationMetadataUpdated(boolean coordinationMetadataUpdated) { + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + return this; + } + + public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { + this.settingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + + public Builder transientSettingsMetadataUpdate(boolean settingsMetadataUpdated) { + this.transientSettingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + + public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { + this.templatesMetadataUpdated = templatesMetadataUpdated; + return this; + } + + public Builder hashesOfConsistentSettingsUpdated(boolean hashesOfConsistentSettingsUpdated) { + this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated; + return this; + } + + public Builder customMetadataUpdated(List customMetadataUpdated) { + this.customMetadataUpdated = customMetadataUpdated; + return this; + } + + public Builder customMetadataDeleted(List customMetadataDeleted) { + this.customMetadataDeleted = customMetadataDeleted; + return this; + } + + public Builder indicesUpdated(List indicesUpdated) { + this.indicesUpdated = indicesUpdated; + return this; + } + + public Builder indicesDeleted(List indicesDeleted) { + this.indicesDeleted = indicesDeleted; + return this; + } + + public Builder clusterBlocksUpdated(boolean clusterBlocksUpdated) { + this.clusterBlocksUpdated = clusterBlocksUpdated; + return this; + } + + public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { + this.discoveryNodesUpdated = discoveryNodesUpdated; + return this; + } + + public Builder clusterStateCustomUpdated(List clusterStateCustomUpdated) { + this.clusterStateCustomUpdated = clusterStateCustomUpdated; + return this; + } + + public Builder clusterStateCustomDeleted(List clusterStateCustomDeleted) { + this.clusterStateCustomDeleted = clusterStateCustomDeleted; + return this; + } + + public ClusterStateDiffManifest build() { + return new ClusterStateDiffManifest( + fromStateUUID, + toStateUUID, + coordinationMetadataUpdated, + settingsMetadataUpdated, + transientSettingsMetadataUpdated, + templatesMetadataUpdated, + customMetadataUpdated, + customMetadataDeleted, + indicesUpdated, + indicesDeleted, + clusterBlocksUpdated, + discoveryNodesUpdated, + hashesOfConsistentSettingsUpdated, + clusterStateCustomUpdated, + clusterStateCustomDeleted + ); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java new file mode 100644 index 0000000000000..4a4b0c79b21a9 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -0,0 +1,134 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import java.io.IOException; +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; +import org.opensearch.gateway.remote.model.RemoteClusterBlocks; +import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms; +import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes; +import org.opensearch.gateway.remote.model.RemoteReadResult; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; + +public class RemoteClusterStateAttributesManager { + public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; + public static final String DISCOVERY_NODES = "nodes"; + public static final String CLUSTER_BLOCKS = "blocks"; + public static final int CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION = 1; + private final RemoteClusterStateBlobStore clusterBlocksBlobStore; + private final RemoteClusterStateBlobStore discoveryNodesBlobStore; + private final RemoteClusterStateBlobStore customsBlobStore; + private final Compressor compressor; + private final NamedXContentRegistry namedXContentRegistry; + + RemoteClusterStateAttributesManager( + RemoteClusterStateBlobStore clusterBlocksBlobStore, RemoteClusterStateBlobStore discoveryNodesBlobStore, RemoteClusterStateBlobStore customsBlobStore, Compressor compressor, NamedXContentRegistry namedXContentRegistry) { + this.clusterBlocksBlobStore = clusterBlocksBlobStore; + this.discoveryNodesBlobStore = discoveryNodesBlobStore; + this.customsBlobStore = customsBlobStore; + this.compressor = compressor; + this.namedXContentRegistry = namedXContentRegistry; + } + + /** + * Allows async upload of Cluster State Attribute components to remote + */ + CheckedRunnable getAsyncMetadataWriteAction( + ClusterState clusterState, + String component, + ToXContent componentData, + LatchedActionListener latchedActionListener + ) { + if (componentData instanceof DiscoveryNodes) { + RemoteDiscoveryNodes remoteObject = new RemoteDiscoveryNodes((DiscoveryNodes)componentData, clusterState.version(), clusterState.metadata().clusterUUID(), compressor, namedXContentRegistry); + return () -> discoveryNodesBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); + } else if (componentData instanceof ClusterBlocks) { + RemoteClusterBlocks remoteObject = new RemoteClusterBlocks((ClusterBlocks) componentData, clusterState.version(), clusterState.metadata().clusterUUID(), compressor, namedXContentRegistry); + return () -> clusterBlocksBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); + } else if (componentData instanceof ClusterState.Custom) { + RemoteClusterStateCustoms remoteObject = new RemoteClusterStateCustoms( + (ClusterState.Custom) componentData, + component, + clusterState.version(), + clusterState.metadata().clusterUUID(), + compressor, + namedXContentRegistry + ); + return () -> customsBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); + } else { + throw new RemoteStateTransferException("Remote object not found for "+ componentData.getClass()); + } + } + + private ActionListener getActionListener(String component, AbstractRemoteWritableBlobEntity remoteObject, LatchedActionListener latchedActionListener) { + return ActionListener.wrap( + resp -> latchedActionListener.onResponse( + remoteObject.getUploadedMetadata() + ), + ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) + ); + } + + public CheckedRunnable getAsyncMetadataReadAction( + String clusterUUID, + String component, + String componentName, + String uploadedFilename, + LatchedActionListener listener + ) { + final ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); + if (component.equals(RemoteDiscoveryNodes.DISCOVERY_NODES)) { + RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); + return () -> discoveryNodesBlobStore.readAsync(remoteDiscoveryNodes, actionListener); + } else if (component.equals(RemoteClusterBlocks.CLUSTER_BLOCKS)) { + RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); + return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); + } else if (component.equals(CLUSTER_STATE_CUSTOM)) { + final ActionListener customActionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, String.join(CUSTOM_DELIMITER, component, componentName))), listener::onFailure); + RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(uploadedFilename, componentName, clusterUUID, compressor, namedXContentRegistry); + return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, customActionListener); + } else { + throw new RemoteStateTransferException("Remote object not found for "+ component); + } + } + + public Map getUpdatedCustoms(ClusterState clusterState, ClusterState previousClusterState) { + Map updatedCustoms = new HashMap<>(); + Set currentCustoms = new HashSet<>(clusterState.customs().keySet()); + for (Map.Entry entry : previousClusterState.customs().entrySet()) { + if (currentCustoms.contains(entry.getKey()) && !entry.getValue().equals(clusterState.customs().get(entry.getKey()))) { + updatedCustoms.put(entry.getKey(), clusterState.customs().get(entry.getKey())); + } + currentCustoms.remove(entry.getKey()); + } + for (String custom : currentCustoms) { + updatedCustoms.put(custom, clusterState.customs().get(custom)); + } + return updatedCustoms; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java new file mode 100644 index 0000000000000..29bde27db150e --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -0,0 +1,102 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +/** + * Wrapper class for uploading/downloading {@link ClusterBlocks} to/from remote blob store + */ +public class RemoteClusterBlocks extends AbstractRemoteWritableBlobEntity { + + public static final String CLUSTER_BLOCKS = "blocks"; + public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( + "blocks", + METADATA_NAME_FORMAT, + ClusterBlocks::fromXContent + ); + + private ClusterBlocks clusterBlocks; + private long stateVersion; + + public RemoteClusterBlocks(final ClusterBlocks clusterBlocks, long stateVersion, String clusterUUID, + final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.clusterBlocks = clusterBlocks; + this.stateVersion = stateVersion; + } + + public RemoteClusterBlocks(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("transient"), CLUSTER_BLOCKS); + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/transient/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(CLUSTER_BLOCKS, blobName); + } + + @Override + public void set(final ClusterBlocks clusterBlocks) { + this.clusterBlocks = clusterBlocks; + } + + @Override + public ClusterBlocks get() { + return clusterBlocks; + } + + + @Override + public InputStream serialize() throws IOException { + return CLUSTER_BLOCKS_FORMAT.serialize(clusterBlocks, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public ClusterBlocks deserialize(final InputStream inputStream) throws IOException { + return CLUSTER_BLOCKS_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java new file mode 100644 index 0000000000000..fffbcfab4e141 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -0,0 +1,108 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; + +public class RemoteClusterStateCustoms extends AbstractRemoteWritableBlobEntity { + public static final String CLUSTER_STATE_CUSTOM = "cluster-state-custom"; + + public final ChecksumBlobStoreFormat clusterStateCustomBlobStoreFormat; + private long stateVersion; + private String customType; + private ClusterState.Custom custom; + + public RemoteClusterStateCustoms(final ClusterState.Custom custom, final String customType, final long stateVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.stateVersion = stateVersion; + this.customType = customType; + this.custom = custom; + this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( + CLUSTER_STATE_CUSTOM, + METADATA_NAME_FORMAT, + parser -> ClusterState.Custom.fromXContent(parser, customType) + ); + } + + public RemoteClusterStateCustoms(final String blobName, final String customType, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.blobName = blobName; + this.customType = customType; + this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( + CLUSTER_STATE_CUSTOM, + METADATA_NAME_FORMAT, + parser -> ClusterState.Custom.fromXContent(parser, customType) + ); + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), CLUSTER_STATE_CUSTOM); + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new ClusterMetadataManifest.UploadedMetadataAttribute(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType), blobName); + } + + @Override + public void set(Custom custom) { + this.custom = custom; + } + + @Override + public ClusterState.Custom get() { + return custom; + } + + @Override + public InputStream serialize() throws IOException { + return clusterStateCustomBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public ClusterState.Custom deserialize(final InputStream inputStream) throws IOException { + return clusterStateCustomBlobStoreFormat.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java new file mode 100644 index 0000000000000..34aca7767a0b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +/** + * Wrapper class for uploading/downloading {@link DiscoveryNodes} to/from remote blob store + */ +public class RemoteDiscoveryNodes extends AbstractRemoteWritableBlobEntity { + + public static final String DISCOVERY_NODES = "nodes"; + public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( + "nodes", + METADATA_NAME_FORMAT, + DiscoveryNodes::fromXContent + ); + + private DiscoveryNodes discoveryNodes; + private long stateVersion; + + public RemoteDiscoveryNodes(final DiscoveryNodes discoveryNodes, final long stateVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.discoveryNodes = discoveryNodes; + this.stateVersion = stateVersion; + } + + public RemoteDiscoveryNodes(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), DISCOVERY_NODES); + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(DISCOVERY_NODES, blobName); + } + + @Override + public void set(final DiscoveryNodes discoveryNodes) { + this.discoveryNodes = discoveryNodes; + } + + @Override + public DiscoveryNodes get() { + return discoveryNodes; + } + + @Override + public InputStream serialize() throws IOException { + return DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public DiscoveryNodes deserialize(final InputStream inputStream) throws IOException { + return DISCOVERY_NODES_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java new file mode 100644 index 0000000000000..3dfa42139bb02 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -0,0 +1,94 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +public class RemoteHashesOfConsistentSettings extends AbstractRemoteWritableBlobEntity { + public static final String HASHES_OF_CONSISTENT_SETTINGS = "hashes-of-consistent-settings"; + public static final ChecksumBlobStoreFormat HASHES_OF_CONSISTENT_SETTINGS_FORMAT = new ChecksumBlobStoreFormat<>( + HASHES_OF_CONSISTENT_SETTINGS, + METADATA_NAME_FORMAT, + DiffableStringMap::fromXContent + ); + + private DiffableStringMap hashesOfConsistentSettings; + private long metadataVersion; + public RemoteHashesOfConsistentSettings(final DiffableStringMap hashesOfConsistentSettings, final long metadataVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.metadataVersion = metadataVersion; + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + } + + public RemoteHashesOfConsistentSettings(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), HASHES_OF_CONSISTENT_SETTINGS); + } + + @Override + public String generateBlobFileName() { + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new ClusterMetadataManifest.UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, blobName); + } + + @Override + public void set(final DiffableStringMap hashesOfConsistentSettings) { + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + } + + @Override + public DiffableStringMap get() { + return hashesOfConsistentSettings; + } + + @Override + public InputStream serialize() throws IOException { + return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.serialize(hashesOfConsistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + } + + @Override + public DiffableStringMap deserialize(final InputStream inputStream) throws IOException { + return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java new file mode 100644 index 0000000000000..adee09eaeffef --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.core.xcontent.ToXContent; + +/** + * Container class for entity read from remote store + */ +public class RemoteReadResult { + + ToXContent obj; + String component; + String componentName; + + public RemoteReadResult(ToXContent obj, String component, String componentName) { + this.obj = obj; + this.component = component; + this.componentName = componentName; + } + + public ToXContent getObj() { + return obj; + } + + public String getComponent() { + return component; + } + + public String getComponentName() { + return componentName; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java new file mode 100644 index 0000000000000..75b6cfc3a765a --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; +import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +/** + * Wrapper class for uploading/downloading transient {@link Settings} to/from remote blob store + */ +public class RemoteTransientSettingsMetadata extends AbstractRemoteWritableBlobEntity { + + public static final String TRANSIENT_SETTING_METADATA = "transient-settings"; + + public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "transient-settings", + METADATA_NAME_FORMAT, + Settings::fromXContent + ); + + private Settings transientSettings; + private long metadataVersion; + + public RemoteTransientSettingsMetadata(final Settings transientSettings, final long metadataVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.transientSettings = transientSettings; + this.metadataVersion = metadataVersion; + } + + public RemoteTransientSettingsMetadata(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + super(clusterUUID, compressor, namedXContentRegistry); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of("global-metadata"), TRANSIENT_SETTING_METADATA); + } + + @Override + public String generateBlobFileName() { + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public void set(final Settings settings) { + this.transientSettings = settings; + } + + @Override + public Settings get() { + return transientSettings; + } + + @Override + public InputStream serialize() throws IOException { + return SETTINGS_METADATA_FORMAT.serialize(transientSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) + .streamInput(); + } + + @Override + public Settings deserialize(final InputStream inputStream) throws IOException { + return SETTINGS_METADATA_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, blobName); + } +} From c83706a7c457112862f7278bf7e9b8759433849b Mon Sep 17 00:00:00 2001 From: Arpit Bandejiya Date: Fri, 7 Jun 2024 14:45:45 +0530 Subject: [PATCH 02/12] Add remote routing table changes in diff Manifest Signed-off-by: Arpit Bandejiya --- .../remote/RemoteRoutingTableService.java | 29 +++++++++ .../remote/ClusterStateDiffManifest.java | 65 +++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index ba2208e17df1f..8e43e1c5f86b6 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -10,9 +10,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.io.IOUtils; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.node.Node; import org.opensearch.node.remotestore.RemoteStoreNodeAttribute; import org.opensearch.repositories.RepositoriesService; @@ -20,6 +25,7 @@ import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.io.IOException; +import java.util.Map; import java.util.function.Supplier; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; @@ -36,12 +42,35 @@ public class RemoteRoutingTableService extends AbstractLifecycleComponent { private final Supplier repositoriesService; private BlobStoreRepository blobStoreRepository; + private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = new DiffableUtils.NonDiffableValueSerializer() { + @Override + public void write(IndexRoutingTable value, StreamOutput out) throws IOException { + value.writeTo(out); + } + + @Override + public IndexRoutingTable read(StreamInput in, String key) throws IOException { + return IndexRoutingTable.readFrom(in); + } + }; + public RemoteRoutingTableService(Supplier repositoriesService, Settings settings) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; this.repositoriesService = repositoriesService; this.settings = settings; } + + public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after) { + return DiffableUtils.diff( + before.getIndicesRouting(), + after.getIndicesRouting(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER + ); + } + + @Override protected void doClose() throws IOException { if (blobStoreRepository != null) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index ea4344d4d35e8..8303f36adde8d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.RoutingTable; +import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; @@ -45,6 +46,7 @@ public class ClusterStateDiffManifest implements ToXContentObject { private static final String DELETES_FIELD = "deletes"; private static final String CLUSTER_BLOCKS_UPDATED_FIELD = "cluster_blocks_diff"; private static final String DISCOVERY_NODES_UPDATED_FIELD = "discovery_nodes_diff"; + private static final String ROUTING_TABLE_DIFF = "routing_table_diff"; private static final String CLUSTER_STATE_CUSTOM_DIFF_FIELD = "cluster_state_custom_diff"; private final String fromStateUUID; @@ -59,6 +61,8 @@ public class ClusterStateDiffManifest implements ToXContentObject { private final List indicesDeleted; private final boolean clusterBlocksUpdated; private final boolean discoveryNodesUpdated; + private final List indicesRoutingUpdated; + private final List indicesRoutingDeleted; private final boolean hashesOfConsistentSettingsUpdated; private final List clusterStateCustomUpdated; private final List clusterStateCustomDeleted; @@ -87,6 +91,13 @@ public class ClusterStateDiffManifest implements ToXContentObject { } } + DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff(previousState.getRoutingTable(), + state.getRoutingTable()); + + indicesRoutingUpdated = new ArrayList<>(); + routingTableDiff.getUpserts().forEach((k,v) -> indicesRoutingUpdated.add(k)); + + indicesRoutingDeleted = routingTableDiff.getDeletes(); hashesOfConsistentSettingsUpdated = !state.metadata().hashesOfConsistentSettings().equals(previousState.metadata().hashesOfConsistentSettings()); clusterStateCustomUpdated = new ArrayList<>(); clusterStateCustomDeleted = new ArrayList<>(); @@ -115,6 +126,8 @@ public ClusterStateDiffManifest( List indicesDeleted, boolean clusterBlocksUpdated, boolean discoveryNodesUpdated, + ListindicesRoutingUpdated, + ListindicesRoutingDeleted, boolean hashesOfConsistentSettingsUpdated, List clusterStateCustomUpdated, List clusterStateCustomDeleted @@ -131,6 +144,8 @@ public ClusterStateDiffManifest( this.indicesDeleted = indicesDeleted; this.clusterBlocksUpdated = clusterBlocksUpdated; this.discoveryNodesUpdated = discoveryNodesUpdated; + this.indicesRoutingUpdated = indicesRoutingUpdated; + this.indicesRoutingDeleted = indicesRoutingDeleted; this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated; this.clusterStateCustomUpdated = clusterStateCustomUpdated; this.clusterStateCustomDeleted = clusterStateCustomDeleted; @@ -176,6 +191,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); + + builder.startObject(ROUTING_TABLE_DIFF); + builder.startArray(UPSERTS_FIELD); + for (String index : indicesRoutingUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String index : indicesRoutingDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); builder.startObject(CLUSTER_STATE_CUSTOM_DIFF_FIELD); builder.startArray(UPSERTS_FIELD); for (String custom : clusterStateCustomUpdated) { @@ -270,6 +298,21 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw throw new XContentParseException("Unexpected token [" + token + "]"); } } + } else if (currentFieldName.equals(ROUTING_TABLE_DIFF)) { + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.indicesRoutingUpdated(parseStringList(parser)); + break; + case DELETES_FIELD: + builder.indicesRoutingDeleted(parseStringList(parser)); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } } else if (currentFieldName.equals(CLUSTER_STATE_CUSTOM_DIFF_FIELD)) { while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { currentFieldName = parser.currentName(); @@ -420,6 +463,14 @@ public boolean isHashesOfConsistentSettingsUpdated() { return hashesOfConsistentSettingsUpdated; } + public List getIndicesRoutingUpdated() { + return indicesRoutingUpdated; + } + + public List getIndicesRoutingDeleted() { + return indicesRoutingDeleted; + } + public List getClusterStateCustomUpdated() { return clusterStateCustomUpdated; } @@ -445,6 +496,8 @@ public static class Builder { private List indicesDeleted; private boolean clusterBlocksUpdated; private boolean discoveryNodesUpdated; + private List indicesRoutingUpdated; + private List indicesRoutingDeleted; private boolean hashesOfConsistentSettingsUpdated; private List clusterStateCustomUpdated; private List clusterStateCustomDeleted; @@ -516,6 +569,16 @@ public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { return this; } + public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { + this.indicesRoutingUpdated = indicesRoutingUpdated; + return this; + } + + public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { + this.indicesRoutingDeleted = indicesRoutingDeleted; + return this; + } + public Builder clusterStateCustomUpdated(List clusterStateCustomUpdated) { this.clusterStateCustomUpdated = clusterStateCustomUpdated; return this; @@ -540,6 +603,8 @@ public ClusterStateDiffManifest build() { indicesDeleted, clusterBlocksUpdated, discoveryNodesUpdated, + indicesRoutingUpdated, + indicesRoutingDeleted, hashesOfConsistentSettingsUpdated, clusterStateCustomUpdated, clusterStateCustomDeleted From 830b00236bd1f149f29a6500c5b3a4aedc9214c1 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Fri, 7 Jun 2024 19:49:54 +0530 Subject: [PATCH 03/12] Use InputStreams rather than XContent for serialization Signed-off-by: Shivansh Arora --- .../cluster/block/ClusterBlocks.java | 16 + .../remote/RemoteRoutingTableService.java | 30 +- .../remote/ClusterMetadataManifest.java | 129 +++++--- .../remote/ClusterStateDiffManifest.java | 81 +++-- .../RemoteClusterStateAttributesManager.java | 101 ++++-- .../remote/RemoteClusterStateService.java | 39 ++- .../remote/RemoteClusterStateUtils.java | 107 ++++++- .../remote/model/RemoteClusterBlocks.java | 62 ++-- .../model/RemoteClusterStateCustoms.java | 74 +++-- .../remote/model/RemoteDiscoveryNodes.java | 57 ++-- .../RemoteHashesOfConsistentSettings.java | 63 ++-- .../RemoteTransientSettingsMetadata.java | 51 ++-- .../cluster/block/ClusterBlockTests.java | 2 +- .../remote/ClusterMetadataManifestTests.java | 287 +++++++++--------- .../model/RemoteClusterBlocksTests.java | 215 +++++++++++++ .../RemoteTransientSettingsMetadataTests.java | 200 ++++++++++++ 16 files changed, 1090 insertions(+), 424 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadataTests.java diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index 02a20b7681ba7..377e8204e9f2f 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -49,6 +49,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Predicate; @@ -325,6 +326,21 @@ public static Diff readDiffFrom(StreamInput in) throws IOExceptio return AbstractDiffable.readDiffFrom(ClusterBlocks::readFrom, in); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterBlocks that = (ClusterBlocks) o; + return Objects.equals(global, that.global) + && Objects.equals(indicesBlocks, that.indicesBlocks) + && Objects.equals(levelHolders, that.levelHolders); + } + + @Override + public int hashCode() { + return Objects.hash(global, indicesBlocks, levelHolders); + } + /** * An immutable level holder. * diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index 8e43e1c5f86b6..058fc3bff772a 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -42,17 +42,18 @@ public class RemoteRoutingTableService extends AbstractLifecycleComponent { private final Supplier repositoriesService; private BlobStoreRepository blobStoreRepository; - private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = new DiffableUtils.NonDiffableValueSerializer() { - @Override - public void write(IndexRoutingTable value, StreamOutput out) throws IOException { - value.writeTo(out); - } - - @Override - public IndexRoutingTable read(StreamInput in, String key) throws IOException { - return IndexRoutingTable.readFrom(in); - } - }; + private static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = + new DiffableUtils.NonDiffableValueSerializer() { + @Override + public void write(IndexRoutingTable value, StreamOutput out) throws IOException { + value.writeTo(out); + } + + @Override + public IndexRoutingTable read(StreamInput in, String key) throws IOException { + return IndexRoutingTable.readFrom(in); + } + }; public RemoteRoutingTableService(Supplier repositoriesService, Settings settings) { assert isRemoteRoutingTableEnabled(settings) : "Remote routing table is not enabled"; @@ -60,8 +61,10 @@ public RemoteRoutingTableService(Supplier repositoriesServi this.settings = settings; } - - public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff(RoutingTable before, RoutingTable after) { + public static DiffableUtils.MapDiff> getIndicesRoutingMapDiff( + RoutingTable before, + RoutingTable after + ) { return DiffableUtils.diff( before.getIndicesRouting(), after.getIndicesRouting(), @@ -70,7 +73,6 @@ public static DiffableUtils.MapDiff ClusterStateDiffManifest.fromXContent(p), DIFF_MANIFEST ); @@ -543,10 +543,26 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.metadataVersion = in.readLong(); - this.uploadedDiscoveryNodesMetadata = new UploadedMetadataAttribute(in); - this.uploadedClusterBlocksMetadata = new UploadedMetadataAttribute(in); - this.uploadedTransientSettingsMetadata = new UploadedMetadataAttribute(in); - this.uploadedHashesOfConsistentSettings = new UploadedMetadataAttribute(in); + if (in.readBoolean()) { + this.uploadedDiscoveryNodesMetadata = new UploadedMetadataAttribute(in); + } else { + this.uploadedDiscoveryNodesMetadata = null; + } + if (in.readBoolean()) { + this.uploadedClusterBlocksMetadata = new UploadedMetadataAttribute(in); + } else { + this.uploadedClusterBlocksMetadata = null; + } + if (in.readBoolean()) { + this.uploadedTransientSettingsMetadata = new UploadedMetadataAttribute(in); + } else { + this.uploadedTransientSettingsMetadata = null; + } + if (in.readBoolean()) { + this.uploadedHashesOfConsistentSettings = new UploadedMetadataAttribute(in); + } else { + this.uploadedHashesOfConsistentSettings = null; + } this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( in.readMap(StreamInput::readString, UploadedMetadataAttribute::new) ); @@ -572,6 +588,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedTransientSettingsMetadata = null; this.uploadedHashesOfConsistentSettings = null; this.uploadedClusterStateCustomMap = null; + } } public static Builder builder() { @@ -695,10 +712,30 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); out.writeLong(metadataVersion); - uploadedDiscoveryNodesMetadata.writeTo(out); - uploadedClusterBlocksMetadata.writeTo(out); - uploadedTransientSettingsMetadata.writeTo(out); - uploadedHashesOfConsistentSettings.writeTo(out); + if (uploadedDiscoveryNodesMetadata != null) { + out.writeBoolean(true); + uploadedDiscoveryNodesMetadata.writeTo(out); + } else { + out.writeBoolean(false); + } + if (uploadedClusterBlocksMetadata != null) { + out.writeBoolean(true); + uploadedClusterBlocksMetadata.writeTo(out); + } else { + out.writeBoolean(false); + } + if (uploadedTransientSettingsMetadata != null) { + out.writeBoolean(true); + uploadedTransientSettingsMetadata.writeTo(out); + } else { + out.writeBoolean(false); + } + if (uploadedHashesOfConsistentSettings != null) { + out.writeBoolean(true); + uploadedHashesOfConsistentSettings.writeTo(out); + } else { + out.writeBoolean(false); + } out.writeMap(uploadedClusterStateCustomMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); @@ -716,30 +753,31 @@ public boolean equals(Object o) { return false; } final ClusterMetadataManifest that = (ClusterMetadataManifest) o; - return Objects.equals(indices, that.indices) - && clusterTerm == that.clusterTerm - && stateVersion == that.stateVersion - && Objects.equals(clusterUUID, that.clusterUUID) - && Objects.equals(stateUUID, that.stateUUID) - && Objects.equals(opensearchVersion, that.opensearchVersion) - && Objects.equals(nodeId, that.nodeId) - && Objects.equals(committed, that.committed) - && Objects.equals(previousClusterUUID, that.previousClusterUUID) - && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) - && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) - && Objects.equals(codecVersion, that.codecVersion) - && Objects.equals(routingTableVersion, that.routingTableVersion) - && Objects.equals(indicesRouting, that.indicesRouting) - && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) - && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) - && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) - && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) - && Objects.equals(metadataVersion, that.metadataVersion) - && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) - && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) - && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) - && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) - && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); + boolean ret = Objects.equals(indices, that.indices); + ret = ret && clusterTerm == that.clusterTerm; + ret = ret && stateVersion == that.stateVersion; + ret = ret && Objects.equals(clusterUUID, that.clusterUUID); + ret = ret && Objects.equals(stateUUID, that.stateUUID); + ret = ret && Objects.equals(opensearchVersion, that.opensearchVersion); + ret = ret && Objects.equals(nodeId, that.nodeId); + ret = ret && Objects.equals(committed, that.committed); + ret = ret && Objects.equals(previousClusterUUID, that.previousClusterUUID); + ret = ret && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted); + ret = ret && Objects.equals(globalMetadataFileName, that.globalMetadataFileName); + ret = ret && Objects.equals(codecVersion, that.codecVersion); + ret = ret && Objects.equals(routingTableVersion, that.routingTableVersion); + ret = ret && Objects.equals(indicesRouting, that.indicesRouting); + ret = ret && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata); + ret = ret && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata); + ret = ret && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata); + ret = ret && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap); + ret = ret && Objects.equals(metadataVersion, that.metadataVersion); + ret = ret && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata); + ret = ret && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata); + ret = ret && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata); + ret = ret && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings); + ret = ret && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); + return ret; } @Override @@ -1233,6 +1271,19 @@ public static UploadedMetadataAttribute fromXContent(XContentParser parser) thro return PARSER.parse(parser, null, parser.currentName()); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UploadedMetadataAttribute that = (UploadedMetadataAttribute) o; + return Objects.equals(attributeName, that.attributeName) && Objects.equals(uploadedFilename, that.uploadedFilename); + } + + @Override + public int hashCode() { + return Objects.hash(attributeName, uploadedFilename); + } + @Override public String toString() { return "UploadedMetadataAttribute{" diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 8303f36adde8d..dd17514211b59 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -24,13 +24,16 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; -import static org.opensearch.core.xcontent.XContentParserUtils.parseStringList; +/** + * Manifest of diff between two cluster states + * + * @opensearch.internal + */ public class ClusterStateDiffManifest implements ToXContentObject { private static final String FROM_STATE_UUID_FIELD = "from_state_uuid"; private static final String TO_STATE_UUID_FIELD = "to_state_uuid"; @@ -80,7 +83,8 @@ public class ClusterStateDiffManifest implements ToXContentObject { discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); customMetadataUpdated = new ArrayList<>(); for (String custom : state.metadata().customs().keySet()) { - if (!previousState.metadata().customs().containsKey(custom) || !state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) { + if (!previousState.metadata().customs().containsKey(custom) + || !state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) { customMetadataUpdated.add(custom); } } @@ -91,14 +95,16 @@ public class ClusterStateDiffManifest implements ToXContentObject { } } - DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService.getIndicesRoutingMapDiff(previousState.getRoutingTable(), - state.getRoutingTable()); + DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService + .getIndicesRoutingMapDiff(previousState.getRoutingTable(), state.getRoutingTable()); indicesRoutingUpdated = new ArrayList<>(); - routingTableDiff.getUpserts().forEach((k,v) -> indicesRoutingUpdated.add(k)); + routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingUpdated.add(k)); indicesRoutingDeleted = routingTableDiff.getDeletes(); - hashesOfConsistentSettingsUpdated = !state.metadata().hashesOfConsistentSettings().equals(previousState.metadata().hashesOfConsistentSettings()); + hashesOfConsistentSettingsUpdated = !state.metadata() + .hashesOfConsistentSettings() + .equals(previousState.metadata().hashesOfConsistentSettings()); clusterStateCustomUpdated = new ArrayList<>(); clusterStateCustomDeleted = new ArrayList<>(); for (String custom : state.customs().keySet()) { @@ -126,8 +132,8 @@ public ClusterStateDiffManifest( List indicesDeleted, boolean clusterBlocksUpdated, boolean discoveryNodesUpdated, - ListindicesRoutingUpdated, - ListindicesRoutingDeleted, + List indicesRoutingUpdated, + List indicesRoutingDeleted, boolean hashesOfConsistentSettingsUpdated, List clusterStateCustomUpdated, List clusterStateCustomDeleted @@ -267,10 +273,10 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw token = parser.nextToken(); switch (currentFieldName) { case UPSERTS_FIELD: - builder.indicesUpdated(parseStringList(parser)); + builder.indicesUpdated(convertListToString(parser.listOrderedMap())); break; case DELETES_FIELD: - builder.indicesDeleted(parseStringList(parser)); + builder.indicesDeleted(convertListToString(parser.listOrderedMap())); break; default: throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); @@ -282,10 +288,10 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw token = parser.nextToken(); switch (currentFieldName) { case UPSERTS_FIELD: - builder.customMetadataUpdated(parseStringList(parser)); + builder.customMetadataUpdated(convertListToString(parser.listOrderedMap())); break; case DELETES_FIELD: - builder.customMetadataDeleted(parseStringList(parser)); + builder.customMetadataDeleted(convertListToString(parser.listOrderedMap())); break; default: throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); @@ -304,10 +310,10 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw parser.nextToken(); switch (currentFieldName) { case UPSERTS_FIELD: - builder.indicesRoutingUpdated(parseStringList(parser)); + builder.indicesRoutingUpdated(convertListToString(parser.listOrderedMap())); break; case DELETES_FIELD: - builder.indicesRoutingDeleted(parseStringList(parser)); + builder.indicesRoutingDeleted(convertListToString(parser.listOrderedMap())); break; default: throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); @@ -319,10 +325,10 @@ public static ClusterStateDiffManifest fromXContent(XContentParser parser) throw parser.nextToken(); switch (currentFieldName) { case UPSERTS_FIELD: - builder.clusterStateCustomUpdated(parseStringList(parser)); + builder.clusterStateCustomUpdated(convertListToString(parser.listOrderedMap())); break; case DELETES_FIELD: - builder.clusterStateCustomDeleted(parseStringList(parser)); + builder.clusterStateCustomDeleted(convertListToString(parser.listOrderedMap())); break; default: throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); @@ -371,6 +377,14 @@ public List findRemovedIndices(Map indices, Map convertListToString(List list) { + List convertedList = new ArrayList<>(); + for (Object o : list) { + convertedList.add(o.toString()); + } + return convertedList; + } + public List findUpdatedIndices(Map indices, Map previousIndices) { List updatedIndices = new ArrayList<>(); for (String index : indices.keySet()) { @@ -383,34 +397,6 @@ public List findUpdatedIndices(Map indices, Map getIndicesRoutingDeleted(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List deletedIndicesRouting = new ArrayList<>(); - for(IndexRoutingTable previousIndexRouting: previousRoutingTable.getIndicesRouting().values()) { - if(!currentRoutingTable.getIndicesRouting().containsKey(previousIndexRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is deleted - deletedIndicesRouting.add(previousIndexRouting.getIndex().getName()); - } - } - return deletedIndicesRouting; - } - - public List getIndicesRoutingUpdated(RoutingTable previousRoutingTable, RoutingTable currentRoutingTable) { - List updatedIndicesRouting = new ArrayList<>(); - for(IndexRoutingTable currentIndicesRouting: currentRoutingTable.getIndicesRouting().values()) { - if(!previousRoutingTable.getIndicesRouting().containsKey(currentIndicesRouting.getIndex().getName())) { - // Latest Routing Table does not have entry for the index which means the index is created - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } else { - if(previousRoutingTable.getIndicesRouting().get(currentIndicesRouting.getIndex().getName()).equals(currentIndicesRouting)) { - // if the latest routing table has the same routing table as the previous routing table, then the index is not updated - continue; - } - updatedIndicesRouting.add(currentIndicesRouting.getIndex().getName()); - } - } - return updatedIndicesRouting; - } - public String getFromStateUUID() { return fromStateUUID; } @@ -483,6 +469,11 @@ public static Builder builder() { return new Builder(); } + /** + * Builder for ClusterStateDiffManifest + * + * @opensearch.internal + */ public static class Builder { private String fromStateUUID; private String toStateUUID; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 4a4b0c79b21a9..67410112b2e60 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -8,33 +8,39 @@ package org.opensearch.gateway.remote; -import java.io.IOException; import org.opensearch.action.LatchedActionListener; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; -import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; -import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.gateway.remote.model.RemoteClusterBlocks; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms; import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes; import org.opensearch.gateway.remote.model.RemoteReadResult; +import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; -import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; +/** + * A Manager which provides APIs to upload and download attributes of ClusterState to the {@link RemoteClusterStateBlobStore} + * + * @opensearch.internal + */ public class RemoteClusterStateAttributesManager { public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; public static final String DISCOVERY_NODES = "nodes"; @@ -45,14 +51,22 @@ public class RemoteClusterStateAttributesManager { private final RemoteClusterStateBlobStore customsBlobStore; private final Compressor compressor; private final NamedXContentRegistry namedXContentRegistry; + private final NamedWriteableRegistry namedWriteableRegistry; RemoteClusterStateAttributesManager( - RemoteClusterStateBlobStore clusterBlocksBlobStore, RemoteClusterStateBlobStore discoveryNodesBlobStore, RemoteClusterStateBlobStore customsBlobStore, Compressor compressor, NamedXContentRegistry namedXContentRegistry) { + RemoteClusterStateBlobStore clusterBlocksBlobStore, + RemoteClusterStateBlobStore discoveryNodesBlobStore, + RemoteClusterStateBlobStore customsBlobStore, + Compressor compressor, + NamedXContentRegistry namedXContentRegistry, + NamedWriteableRegistry namedWriteableRegistry + ) { this.clusterBlocksBlobStore = clusterBlocksBlobStore; this.discoveryNodesBlobStore = discoveryNodesBlobStore; this.customsBlobStore = customsBlobStore; this.compressor = compressor; this.namedXContentRegistry = namedXContentRegistry; + this.namedWriteableRegistry = namedWriteableRegistry; } /** @@ -65,10 +79,25 @@ CheckedRunnable getAsyncMetadataWriteAction( LatchedActionListener latchedActionListener ) { if (componentData instanceof DiscoveryNodes) { - RemoteDiscoveryNodes remoteObject = new RemoteDiscoveryNodes((DiscoveryNodes)componentData, clusterState.version(), clusterState.metadata().clusterUUID(), compressor, namedXContentRegistry); - return () -> discoveryNodesBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); + RemoteDiscoveryNodes remoteObject = new RemoteDiscoveryNodes( + (DiscoveryNodes) componentData, + clusterState.version(), + clusterState.metadata().clusterUUID(), + compressor, + namedXContentRegistry + ); + return () -> discoveryNodesBlobStore.writeAsync( + remoteObject, + getActionListener(component, remoteObject, latchedActionListener) + ); } else if (componentData instanceof ClusterBlocks) { - RemoteClusterBlocks remoteObject = new RemoteClusterBlocks((ClusterBlocks) componentData, clusterState.version(), clusterState.metadata().clusterUUID(), compressor, namedXContentRegistry); + RemoteClusterBlocks remoteObject = new RemoteClusterBlocks( + (ClusterBlocks) componentData, + clusterState.version(), + clusterState.metadata().clusterUUID(), + compressor, + namedXContentRegistry + ); return () -> clusterBlocksBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else if (componentData instanceof ClusterState.Custom) { RemoteClusterStateCustoms remoteObject = new RemoteClusterStateCustoms( @@ -77,19 +106,22 @@ CheckedRunnable getAsyncMetadataWriteAction( clusterState.version(), clusterState.metadata().clusterUUID(), compressor, - namedXContentRegistry + namedXContentRegistry, + namedWriteableRegistry ); return () -> customsBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); } else { - throw new RemoteStateTransferException("Remote object not found for "+ componentData.getClass()); + throw new RemoteStateTransferException("Remote object not found for " + componentData.getClass()); } } - private ActionListener getActionListener(String component, AbstractRemoteWritableBlobEntity remoteObject, LatchedActionListener latchedActionListener) { + private ActionListener getActionListener( + String component, + AbstractRemoteWritableBlobEntity remoteObject, + LatchedActionListener latchedActionListener + ) { return ActionListener.wrap( - resp -> latchedActionListener.onResponse( - remoteObject.getUploadedMetadata() - ), + resp -> latchedActionListener.onResponse(remoteObject.getUploadedMetadata()), ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) ); } @@ -101,19 +133,48 @@ public CheckedRunnable getAsyncMetadataReadAction( String uploadedFilename, LatchedActionListener listener ) { - final ActionListener actionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure); + final ActionListener actionListener = ActionListener.wrap( + response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), + listener::onFailure + ); if (component.equals(RemoteDiscoveryNodes.DISCOVERY_NODES)) { - RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); + RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes( + uploadedFilename, + clusterUUID, + compressor, + namedXContentRegistry + ); return () -> discoveryNodesBlobStore.readAsync(remoteDiscoveryNodes, actionListener); } else if (component.equals(RemoteClusterBlocks.CLUSTER_BLOCKS)) { - RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks(uploadedFilename, clusterUUID, compressor, namedXContentRegistry); + RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks( + uploadedFilename, + clusterUUID, + compressor, + namedXContentRegistry + ); return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); } else if (component.equals(CLUSTER_STATE_CUSTOM)) { - final ActionListener customActionListener = ActionListener.wrap(response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, String.join(CUSTOM_DELIMITER, component, componentName))), listener::onFailure); - RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms(uploadedFilename, componentName, clusterUUID, compressor, namedXContentRegistry); + final ActionListener customActionListener = ActionListener.wrap( + response -> listener.onResponse( + new RemoteReadResult( + (ToXContent) response, + CLUSTER_STATE_ATTRIBUTE, + String.join(CUSTOM_DELIMITER, component, componentName) + ) + ), + listener::onFailure + ); + RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms( + uploadedFilename, + componentName, + clusterUUID, + compressor, + namedXContentRegistry, + namedWriteableRegistry + ); return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, customActionListener); } else { - throw new RemoteStateTransferException("Remote object not found for "+ component); + throw new RemoteStateTransferException("Remote object not found for " + component); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index d0593dcd51475..50487315d5867 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -790,27 +790,24 @@ private ClusterMetadataManifest uploadManifest( committed, MANIFEST_CURRENT_CODEC_VERSION ); - final ClusterMetadataManifest manifest = new ClusterMetadataManifest( - clusterState.term(), - clusterState.getVersion(), - clusterState.metadata().clusterUUID(), - clusterState.stateUUID(), - Version.CURRENT, - nodeId, - committed, - MANIFEST_CURRENT_CODEC_VERSION, - null, - uploadedIndexMetadata, - previousClusterUUID, - clusterState.metadata().clusterUUIDCommitted(), - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplatesMetadata, - uploadedCustomMetadataMap, - clusterState.routingTable().version(), - // TODO: Add actual list of changed indices routing with index routing upload flow. - new ArrayList<>() - ); + final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(clusterState.term()) + .stateVersion(clusterState.getVersion()) + .clusterUUID(clusterState.metadata().clusterUUID()) + .stateUUID(clusterState.stateUUID()) + .opensearchVersion(Version.CURRENT) + .nodeId(nodeId) + .committed(committed) + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .indices(uploadedIndexMetadata) + .previousClusterUUID(previousClusterUUID) + .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) + .coordinationMetadata(uploadedCoordinationMetadata) + .settingMetadata(uploadedSettingsMetadata) + .templatesMetadata(uploadedTemplatesMetadata) + .customMetadataMap(uploadedCustomMetadataMap) + .routingTableVersion(clusterState.routingTable().version()) + .build(); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return manifest; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index f87b0a6e401d3..66d7ac586a59d 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -9,10 +9,17 @@ package org.opensearch.gateway.remote; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; import java.util.Map; /** @@ -21,11 +28,14 @@ public class RemoteClusterStateUtils { public static final String DELIMITER = "__"; - public static final String PATH_DELIMITER = "/"; - public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; public static final String METADATA_NAME_FORMAT = "%s.dat"; - public static final String METADATA_NAME_PLAIN_FORMAT = "%s"; + public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; + public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; + public static final String CLUSTER_STATE_EPHEMERAL_PATH_TOKEN = "ephemeral"; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; + public static final String CUSTOM_DELIMITER = "--"; + public static final String PATH_DELIMITER = "/"; + public static final String METADATA_NAME_PLAIN_FORMAT = "%s"; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. @@ -33,7 +43,98 @@ public class RemoteClusterStateUtils { Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY) ); + public static BlobPath getCusterMetadataBasePath(BlobStoreRepository blobStoreRepository, String clusterName, String clusterUUID) { + return blobStoreRepository.basePath().add(encodeString(clusterName)).add(CLUSTER_STATE_PATH_TOKEN).add(clusterUUID); + } + public static String encodeString(String content) { return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8)); } + + public static String getFormattedFileName(String fileName, int codecVersion) { + if (codecVersion < ClusterMetadataManifest.CODEC_V3) { + return String.format(Locale.ROOT, METADATA_NAME_FORMAT, fileName); + } + return fileName; + } + + static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepository, String clusterName) { + return blobStoreRepository.blobStore() + .blobContainer( + blobStoreRepository.basePath() + .add(Base64.getUrlEncoder().withoutPadding().encodeToString(clusterName.getBytes(StandardCharsets.UTF_8))) + .add(CLUSTER_STATE_PATH_TOKEN) + ); + } + + /** + * Exception for Remote state transfer. + */ + public static class RemoteStateTransferException extends RuntimeException { + + public RemoteStateTransferException(String errorDesc) { + super(errorDesc); + } + + public RemoteStateTransferException(String errorDesc, Throwable cause) { + super(errorDesc, cause); + } + } + + /** + * Container class to keep metadata of all uploaded attributes + */ + public static class UploadedMetadataResults { + List uploadedIndexMetadata; + Map uploadedCustomMetadataMap; + Map uploadedClusterStateCustomMetadataMap; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks; + List uploadedIndicesRoutingMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedHashesOfConsistentSettings; + + public UploadedMetadataResults( + List uploadedIndexMetadata, + Map uploadedCustomMetadataMap, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks, + List uploadedIndicesRoutingMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedHashesOfConsistentSettings, + Map uploadedClusterStateCustomMap + ) { + this.uploadedIndexMetadata = uploadedIndexMetadata; + this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; + this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; + this.uploadedSettingsMetadata = uploadedSettingsMetadata; + this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata; + this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + this.uploadedDiscoveryNodes = uploadedDiscoveryNodes; + this.uploadedClusterBlocks = uploadedClusterBlocks; + this.uploadedIndicesRoutingMetadata = uploadedIndicesRoutingMetadata; + this.uploadedHashesOfConsistentSettings = uploadedHashesOfConsistentSettings; + this.uploadedClusterStateCustomMetadataMap = uploadedClusterStateCustomMap; + } + + public UploadedMetadataResults() { + this.uploadedIndexMetadata = new ArrayList<>(); + this.uploadedCustomMetadataMap = new HashMap<>(); + this.uploadedCoordinationMetadata = null; + this.uploadedSettingsMetadata = null; + this.uploadedTransientSettingsMetadata = null; + this.uploadedTemplatesMetadata = null; + this.uploadedDiscoveryNodes = null; + this.uploadedClusterBlocks = null; + this.uploadedIndicesRoutingMetadata = new ArrayList<>(); + this.uploadedHashesOfConsistentSettings = null; + this.uploadedClusterStateCustomMetadataMap = new HashMap<>(); + } + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index 29bde27db150e..779961058350f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -8,24 +8,27 @@ package org.opensearch.gateway.remote.model; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; -import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.core.common.bytes.BytesReference.toBytes; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; /** * Wrapper class for uploading/downloading {@link ClusterBlocks} to/from remote blob store @@ -33,30 +36,35 @@ public class RemoteClusterBlocks extends AbstractRemoteWritableBlobEntity { public static final String CLUSTER_BLOCKS = "blocks"; - public static final ChecksumBlobStoreFormat CLUSTER_BLOCKS_FORMAT = new ChecksumBlobStoreFormat<>( - "blocks", - METADATA_NAME_FORMAT, - ClusterBlocks::fromXContent - ); private ClusterBlocks clusterBlocks; private long stateVersion; - public RemoteClusterBlocks(final ClusterBlocks clusterBlocks, long stateVersion, String clusterUUID, - final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteClusterBlocks( + final ClusterBlocks clusterBlocks, + long stateVersion, + String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.clusterBlocks = clusterBlocks; this.stateVersion = stateVersion; } - public RemoteClusterBlocks(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteClusterBlocks( + final String blobName, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; } @Override public BlobPathParameters getBlobPathParameters() { - return new BlobPathParameters(List.of("transient"), CLUSTER_BLOCKS); + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), CLUSTER_BLOCKS); } @Override @@ -79,24 +87,16 @@ public UploadedMetadata getUploadedMetadata() { return new UploadedMetadataAttribute(CLUSTER_BLOCKS, blobName); } - @Override - public void set(final ClusterBlocks clusterBlocks) { - this.clusterBlocks = clusterBlocks; - } - - @Override - public ClusterBlocks get() { - return clusterBlocks; - } - - @Override public InputStream serialize() throws IOException { - return CLUSTER_BLOCKS_FORMAT.serialize(clusterBlocks, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + clusterBlocks.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); } @Override public ClusterBlocks deserialize(final InputStream inputStream) throws IOException { - return CLUSTER_BLOCKS_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream))); + return ClusterBlocks.readFrom(in); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java index fffbcfab4e141..e3347d3926a88 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -11,54 +11,67 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterState.Custom; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; -import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; import java.io.IOException; import java.io.InputStream; import java.util.List; +import static org.opensearch.cluster.ClusterState.FeatureAware.shouldSerialize; +import static org.opensearch.core.common.bytes.BytesReference.toBytes; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; -import static org.opensearch.gateway.remote.model.RemoteCustomMetadata.CUSTOM_DELIMITER; +/** + * Wrapper class for uploading/downloading {@link Custom} to/from remote blob store + */ public class RemoteClusterStateCustoms extends AbstractRemoteWritableBlobEntity { public static final String CLUSTER_STATE_CUSTOM = "cluster-state-custom"; - public final ChecksumBlobStoreFormat clusterStateCustomBlobStoreFormat; private long stateVersion; private String customType; private ClusterState.Custom custom; + private final NamedWriteableRegistry namedWriteableRegistry; - public RemoteClusterStateCustoms(final ClusterState.Custom custom, final String customType, final long stateVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteClusterStateCustoms( + final ClusterState.Custom custom, + final String customType, + final long stateVersion, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry, + final NamedWriteableRegistry namedWriteableRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.stateVersion = stateVersion; this.customType = customType; this.custom = custom; - this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( - CLUSTER_STATE_CUSTOM, - METADATA_NAME_FORMAT, - parser -> ClusterState.Custom.fromXContent(parser, customType) - ); + this.namedWriteableRegistry = namedWriteableRegistry; } - public RemoteClusterStateCustoms(final String blobName, final String customType, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteClusterStateCustoms( + final String blobName, + final String customType, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry, + final NamedWriteableRegistry namedWriteableRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; this.customType = customType; - this.clusterStateCustomBlobStoreFormat = new ChecksumBlobStoreFormat<>( - CLUSTER_STATE_CUSTOM, - METADATA_NAME_FORMAT, - parser -> ClusterState.Custom.fromXContent(parser, customType) - ); + this.namedWriteableRegistry = namedWriteableRegistry; } @Override @@ -83,26 +96,27 @@ public String generateBlobFileName() { @Override public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { assert blobName != null; - return new ClusterMetadataManifest.UploadedMetadataAttribute(String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType), blobName); - } - - @Override - public void set(Custom custom) { - this.custom = custom; - } - - @Override - public ClusterState.Custom get() { - return custom; + return new ClusterMetadataManifest.UploadedMetadataAttribute( + String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType), + blobName + ); } @Override public InputStream serialize() throws IOException { - return clusterStateCustomBlobStoreFormat.serialize(custom, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput outputStream = new BytesStreamOutput(); + if (shouldSerialize(outputStream, custom)) { + outputStream.writeNamedWriteable(custom); + } + return outputStream.bytes().streamInput(); } @Override public ClusterState.Custom deserialize(final InputStream inputStream) throws IOException { - return clusterStateCustomBlobStoreFormat.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput( + new BytesStreamInput(toBytes(Streams.readFully(inputStream))), + this.namedWriteableRegistry + ); + return in.readNamedWriteable(Custom.class); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index 34aca7767a0b1..9bf0ca95ac9a9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -8,25 +8,26 @@ package org.opensearch.gateway.remote.model; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; -import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.core.common.bytes.BytesReference.toBytes; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; /** * Wrapper class for uploading/downloading {@link DiscoveryNodes} to/from remote blob store @@ -34,22 +35,28 @@ public class RemoteDiscoveryNodes extends AbstractRemoteWritableBlobEntity { public static final String DISCOVERY_NODES = "nodes"; - public static final ChecksumBlobStoreFormat DISCOVERY_NODES_FORMAT = new ChecksumBlobStoreFormat<>( - "nodes", - METADATA_NAME_FORMAT, - DiscoveryNodes::fromXContent - ); private DiscoveryNodes discoveryNodes; private long stateVersion; - public RemoteDiscoveryNodes(final DiscoveryNodes discoveryNodes, final long stateVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteDiscoveryNodes( + final DiscoveryNodes discoveryNodes, + final long stateVersion, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.discoveryNodes = discoveryNodes; this.stateVersion = stateVersion; } - public RemoteDiscoveryNodes(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteDiscoveryNodes( + final String blobName, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; } @@ -79,23 +86,15 @@ public UploadedMetadata getUploadedMetadata() { return new UploadedMetadataAttribute(DISCOVERY_NODES, blobName); } - @Override - public void set(final DiscoveryNodes discoveryNodes) { - this.discoveryNodes = discoveryNodes; - } - - @Override - public DiscoveryNodes get() { - return discoveryNodes; - } - @Override public InputStream serialize() throws IOException { - return DISCOVERY_NODES_FORMAT.serialize(discoveryNodes, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput outputStream = new BytesStreamOutput(); + discoveryNodes.writeTo(outputStream); + return outputStream.bytes().streamInput(); } @Override public DiscoveryNodes deserialize(final InputStream inputStream) throws IOException { - return DISCOVERY_NODES_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + return DiscoveryNodes.readFrom(new BytesStreamInput(toBytes(Streams.readFully(inputStream))), null); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java index 3dfa42139bb02..4ece6404e7199 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -8,42 +8,54 @@ package org.opensearch.gateway.remote.model; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; import org.opensearch.cluster.metadata.DiffableStringMap; import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest; -import org.opensearch.gateway.remote.RemoteClusterStateUtils; import org.opensearch.index.remote.RemoteStoreUtils; -import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.core.common.bytes.BytesReference.toBytes; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; + +/** + * Wrapper class for uploading/downloading {@link DiffableStringMap} to/from remote blob store + */ public class RemoteHashesOfConsistentSettings extends AbstractRemoteWritableBlobEntity { public static final String HASHES_OF_CONSISTENT_SETTINGS = "hashes-of-consistent-settings"; - public static final ChecksumBlobStoreFormat HASHES_OF_CONSISTENT_SETTINGS_FORMAT = new ChecksumBlobStoreFormat<>( - HASHES_OF_CONSISTENT_SETTINGS, - METADATA_NAME_FORMAT, - DiffableStringMap::fromXContent - ); private DiffableStringMap hashesOfConsistentSettings; private long metadataVersion; - public RemoteHashesOfConsistentSettings(final DiffableStringMap hashesOfConsistentSettings, final long metadataVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + + public RemoteHashesOfConsistentSettings( + final DiffableStringMap hashesOfConsistentSettings, + final long metadataVersion, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.metadataVersion = metadataVersion; this.hashesOfConsistentSettings = hashesOfConsistentSettings; } - public RemoteHashesOfConsistentSettings(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteHashesOfConsistentSettings( + final String blobName, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; } @@ -72,23 +84,16 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { return new ClusterMetadataManifest.UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, blobName); } - @Override - public void set(final DiffableStringMap hashesOfConsistentSettings) { - this.hashesOfConsistentSettings = hashesOfConsistentSettings; - } - - @Override - public DiffableStringMap get() { - return hashesOfConsistentSettings; - } - @Override public InputStream serialize() throws IOException { - return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.serialize(hashesOfConsistentSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS).streamInput(); + BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); + hashesOfConsistentSettings.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); } @Override public DiffableStringMap deserialize(final InputStream inputStream) throws IOException { - return HASHES_OF_CONSISTENT_SETTINGS_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream))); + return DiffableStringMap.readFrom(in); } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java index 75b6cfc3a765a..fe32b95f5e957 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java @@ -8,13 +8,6 @@ package org.opensearch.gateway.remote.model; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; -import static org.opensearch.gateway.remote.RemoteGlobalMetadataManager.GLOBAL_METADATA_CURRENT_CODEC_VERSION; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; import org.opensearch.common.io.Streams; import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; @@ -27,6 +20,15 @@ import org.opensearch.index.remote.RemoteStoreUtils; import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + /** * Wrapper class for uploading/downloading transient {@link Settings} to/from remote blob store */ @@ -43,20 +45,31 @@ public class RemoteTransientSettingsMetadata extends AbstractRemoteWritableBlobE private Settings transientSettings; private long metadataVersion; - public RemoteTransientSettingsMetadata(final Settings transientSettings, final long metadataVersion, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteTransientSettingsMetadata( + final Settings transientSettings, + final long metadataVersion, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.transientSettings = transientSettings; this.metadataVersion = metadataVersion; } - public RemoteTransientSettingsMetadata(final String blobName, final String clusterUUID, final Compressor compressor, final NamedXContentRegistry namedXContentRegistry) { + public RemoteTransientSettingsMetadata( + final String blobName, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { super(clusterUUID, compressor, namedXContentRegistry); this.blobName = blobName; } @Override public BlobPathParameters getBlobPathParameters() { - return new BlobPathParameters(List.of("global-metadata"), TRANSIENT_SETTING_METADATA); + return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), TRANSIENT_SETTING_METADATA); } @Override @@ -72,20 +85,14 @@ public String generateBlobFileName() { return blobFileName; } - @Override - public void set(final Settings settings) { - this.transientSettings = settings; - } - - @Override - public Settings get() { - return transientSettings; - } - @Override public InputStream serialize() throws IOException { - return SETTINGS_METADATA_FORMAT.serialize(transientSettings, generateBlobFileName(), getCompressor(), RemoteClusterStateUtils.FORMAT_PARAMS) - .streamInput(); + return SETTINGS_METADATA_FORMAT.serialize( + transientSettings, + generateBlobFileName(), + getCompressor(), + RemoteClusterStateUtils.FORMAT_PARAMS + ).streamInput(); } @Override diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java index 04e04bd96a7d3..0f049f99b49c0 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java @@ -136,7 +136,7 @@ public void testGetIndexBlockWithId() { assertThat(builder.build().getIndexBlockWithId("index", randomValueOtherThan(blockId, OpenSearchTestCase::randomInt)), nullValue()); } - private ClusterBlock randomClusterBlock() { + public static ClusterBlock randomClusterBlock() { final String uuid = randomBoolean() ? UUIDs.randomBase64UUID() : null; final List levels = Arrays.asList(ClusterBlockLevel.values()); return new ClusterBlock( diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index d1f559eb75f85..e868004f90d60 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -33,6 +33,7 @@ import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; public class ClusterMetadataManifestTests extends OpenSearchTestCase { @@ -91,42 +92,43 @@ public void testClusterMetadataManifestXContentV1() throws IOException { public void testClusterMetadataManifestXContent() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V3, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V3) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file")) + .settingMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file")) + .templatesMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file")) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 1L, - randomUploadedIndexMetadataList() - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -139,42 +141,45 @@ public void testClusterMetadataManifestXContent() throws IOException { } public void testClusterMetadataManifestSerializationEqualsHashCode() { - ClusterMetadataManifest initialManifest = new ClusterMetadataManifest( - 1337L, - 7L, - "HrYF3kP5SmSPWtKlWhnNSA", - "6By9p9G0Rv2MmFYJcPAOgA", - Version.CURRENT, - "B10RX1f5RJenMQvYccCgSQ", - true, - 2, - null, - randomUploadedIndexMetadataList(), - "yfObdx8KSMKKrXf8UyHhM", - true, - new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder() + .clusterTerm(1337L) + .stateVersion(7L) + .clusterUUID("HrYF3kP5SmSPWtKlWhnNSA") + .stateUUID("6By9p9G0Rv2MmFYJcPAOgA") + .opensearchVersion(Version.CURRENT) + .nodeId("B10RX1f5RJenMQvYccCgSQ") + .committed(true) + .codecVersion(ClusterMetadataManifest.CODEC_V3) + .indices(randomUploadedIndexMetadataList()) + .previousClusterUUID("yfObdx8KSMKKrXf8UyHhM") + .clusterUUIDCommitted(true) + .coordinationMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file")) + .settingMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file")) + .templatesMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file")) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 1L, - randomUploadedIndexMetadataList() - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, "discovery-nodes-file")) + + .build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialManifest, @@ -316,42 +321,42 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { public void testClusterMetadataManifestXContentV2() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V2, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V2) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(uploadedMetadataAttribute) + .settingMetadata(uploadedMetadataAttribute) + .templatesMetadata(uploadedMetadataAttribute) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 0, - new ArrayList<>() - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -366,42 +371,44 @@ public void testClusterMetadataManifestXContentV2() throws IOException { public void testClusterMetadataManifestXContentV3() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V3, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V3) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(uploadedMetadataAttribute) + .settingMetadata(uploadedMetadataAttribute) + .templatesMetadata(uploadedMetadataAttribute) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 1L, - Collections.singletonList(uploadedIndexMetadata) - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java new file mode 100644 index 0000000000000..fffdffa0c2373 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java @@ -0,0 +1,215 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.cluster.block.ClusterBlockTests.randomClusterBlock; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; + +public class RemoteClusterBlocksTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( + clusterBlocks, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( + clusterBlocks, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( + clusterBlocks, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/cluster-blocks"; + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(uploadedFile, clusterUUID, compressor, namedXContentRegistry); + assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "cluster-blocks" }); + } + + public void testBlobPathParameters() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( + clusterBlocks, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), CLUSTER_BLOCKS); + } + + public void testGenerateBlobFileName() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( + clusterBlocks, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], CLUSTER_BLOCKS); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION)); + + } + + public void testGetUploadedMetadata() throws IOException { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( + clusterBlocks, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), CLUSTER_BLOCKS); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + } + + public void testSerDe() throws IOException { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( + clusterBlocks, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + ClusterBlocks readClusterBlocks = remoteObjectForUpload.deserialize(inputStream); + assertEquals(clusterBlocks.global(), readClusterBlocks.global()); + assertEquals(clusterBlocks.indices().keySet(), readClusterBlocks.indices().keySet()); + for (String index : clusterBlocks.indices().keySet()) { + assertEquals(clusterBlocks.indices().get(index), readClusterBlocks.indices().get(index)); + } + + } + } + + private ClusterBlocks randomClusterBlocks() { + ClusterBlocks.Builder builder = ClusterBlocks.builder(); + int randomGlobalBlocks = randomIntBetween(0, 10); + for (int i = 0; i < randomGlobalBlocks; i++) { + builder.addGlobalBlock(randomClusterBlock()); + } + + int randomIndices = randomIntBetween(0, 10); + for (int i = 0; i < randomIndices; i++) { + int randomIndexBlocks = randomIntBetween(0, 10); + for (int j = 0; j < randomIndexBlocks; j++) { + builder.addIndexBlock("index-" + i, randomClusterBlock()); + } + } + return builder.build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadataTests.java new file mode 100644 index 0000000000000..4061ab4a6b5ef --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadataTests.java @@ -0,0 +1,200 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterModule; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; + +public class RemoteTransientSettingsMetadataTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/settings"; + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + uploadedFile, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "settings" }); + } + + public void testBlobPathParameters() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), TRANSIENT_SETTING_METADATA); + } + + public void testGenerateBlobFileName() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], TRANSIENT_SETTING_METADATA); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION)); + + } + + public void testGetUploadedMetadata() throws IOException { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), TRANSIENT_SETTING_METADATA); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + } + + public void testSerDe() throws IOException { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + Settings readsettings = remoteObjectForUpload.deserialize(inputStream); + assertEquals(readsettings, settings); + } + } + + private Settings getSettings() { + return Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build(); + } +} From da6580f092ff6185f7005fcf7948ec5e717cdc85 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Sat, 8 Jun 2024 16:47:25 +0530 Subject: [PATCH 04/12] Address PR comments Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 53 ++++++++++--------- .../remote/ClusterStateDiffManifest.java | 1 - .../RemoteClusterStateAttributesManager.java | 2 +- .../remote/RemoteClusterStateUtils.java | 12 +++++ .../remote/model/RemoteClusterBlocks.java | 16 ++++-- .../model/RemoteClusterStateCustoms.java | 26 +++++---- .../remote/model/RemoteDiscoveryNodes.java | 16 ++++-- .../RemoteHashesOfConsistentSettings.java | 17 ++++-- 8 files changed, 92 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 8a91522136bbb..1f2fe4343bc05 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -753,36 +753,34 @@ public boolean equals(Object o) { return false; } final ClusterMetadataManifest that = (ClusterMetadataManifest) o; - boolean ret = Objects.equals(indices, that.indices); - ret = ret && clusterTerm == that.clusterTerm; - ret = ret && stateVersion == that.stateVersion; - ret = ret && Objects.equals(clusterUUID, that.clusterUUID); - ret = ret && Objects.equals(stateUUID, that.stateUUID); - ret = ret && Objects.equals(opensearchVersion, that.opensearchVersion); - ret = ret && Objects.equals(nodeId, that.nodeId); - ret = ret && Objects.equals(committed, that.committed); - ret = ret && Objects.equals(previousClusterUUID, that.previousClusterUUID); - ret = ret && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted); - ret = ret && Objects.equals(globalMetadataFileName, that.globalMetadataFileName); - ret = ret && Objects.equals(codecVersion, that.codecVersion); - ret = ret && Objects.equals(routingTableVersion, that.routingTableVersion); - ret = ret && Objects.equals(indicesRouting, that.indicesRouting); - ret = ret && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata); - ret = ret && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata); - ret = ret && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata); - ret = ret && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap); - ret = ret && Objects.equals(metadataVersion, that.metadataVersion); - ret = ret && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata); - ret = ret && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata); - ret = ret && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata); - ret = ret && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings); - ret = ret && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); - return ret; + return Objects.equals(indices, that.indices) + && clusterTerm == that.clusterTerm + && stateVersion == that.stateVersion + && Objects.equals(clusterUUID, that.clusterUUID) + && Objects.equals(stateUUID, that.stateUUID) + && Objects.equals(opensearchVersion, that.opensearchVersion) + && Objects.equals(nodeId, that.nodeId) + && Objects.equals(committed, that.committed) + && Objects.equals(previousClusterUUID, that.previousClusterUUID) + && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) + && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) + && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(routingTableVersion, that.routingTableVersion) + && Objects.equals(indicesRouting, that.indicesRouting) + && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) + && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) + && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) + && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) + && Objects.equals(metadataVersion, that.metadataVersion) + && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) + && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) + && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) + && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) + && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); } @Override public int hashCode() { - // ToDo: update this method to contain new attributes return Objects.hash( codecVersion, globalMetadataFileName, @@ -1033,6 +1031,9 @@ public Builder(ClusterMetadataManifest manifest) { this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; this.routingTableVersion = manifest.routingTableVersion; this.indicesRouting = new ArrayList<>(manifest.indicesRouting); + this.discoveryNodesMetadata = manifest.uploadedDiscoveryNodesMetadata; + this.clusterBlocksMetadata = manifest.uploadedClusterBlocksMetadata; + this.transientSettingsMetadata = manifest.uploadedTransientSettingsMetadata; this.diffManifest = manifest.diffManifest; this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings; this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap; diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index dd17514211b59..9821f5c0b85da 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -13,7 +13,6 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.Metadata; import org.opensearch.cluster.routing.IndexRoutingTable; -import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.MediaTypeRegistry; diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 67410112b2e60..100a26b3bdc5a 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -122,7 +122,7 @@ private ActionListener getActionListener( ) { return ActionListener.wrap( resp -> latchedActionListener.onResponse(remoteObject.getUploadedMetadata()), - ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, ex)) + ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, remoteObject, ex)) ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 66d7ac586a59d..4e260725648b4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -11,6 +11,7 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.remote.RemoteWriteableEntity; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.repositories.blobstore.BlobStoreRepository; @@ -71,6 +72,7 @@ static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepositor * Exception for Remote state transfer. */ public static class RemoteStateTransferException extends RuntimeException { + private RemoteWriteableEntity entity; public RemoteStateTransferException(String errorDesc) { super(errorDesc); @@ -79,6 +81,16 @@ public RemoteStateTransferException(String errorDesc) { public RemoteStateTransferException(String errorDesc, Throwable cause) { super(errorDesc, cause); } + + public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity) { + super(errorDesc); + this.entity = entity; + } + + public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity, Throwable cause) { + super(errorDesc, cause); + this.entity = entity; + } } /** diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index 779961058350f..9bb0eac5bd793 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -89,14 +89,20 @@ public UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - clusterBlocks.writeTo(bytesStreamOutput); - return bytesStreamOutput.bytes().streamInput(); + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { + clusterBlocks.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize remote cluster blocks", e); + } } @Override public ClusterBlocks deserialize(final InputStream inputStream) throws IOException { - StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream))); - return ClusterBlocks.readFrom(in); + try (StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream)))) { + return ClusterBlocks.readFrom(in); + } catch (IOException e) { + throw new IOException("Failed to deserialize remote cluster blocks", e); + } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java index e3347d3926a88..638de65091b09 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -104,19 +104,27 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - BytesStreamOutput outputStream = new BytesStreamOutput(); - if (shouldSerialize(outputStream, custom)) { - outputStream.writeNamedWriteable(custom); + try (BytesStreamOutput outputStream = new BytesStreamOutput()) { + if (shouldSerialize(outputStream, custom)) { + outputStream.writeNamedWriteable(custom); + } + return outputStream.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize cluster state custom of type " + customType, e); } - return outputStream.bytes().streamInput(); } @Override public ClusterState.Custom deserialize(final InputStream inputStream) throws IOException { - NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput( - new BytesStreamInput(toBytes(Streams.readFully(inputStream))), - this.namedWriteableRegistry - ); - return in.readNamedWriteable(Custom.class); + try ( + NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput( + new BytesStreamInput(toBytes(Streams.readFully(inputStream))), + this.namedWriteableRegistry + ) + ) { + return in.readNamedWriteable(Custom.class); + } catch (IOException e) { + throw new IOException("Failed to deserialize cluster state custom of type " + customType, e); + } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java index 9bf0ca95ac9a9..63e0685eebb10 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -14,6 +14,7 @@ import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; import org.opensearch.common.remote.BlobPathParameters; import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; @@ -88,13 +89,20 @@ public UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - BytesStreamOutput outputStream = new BytesStreamOutput(); - discoveryNodes.writeTo(outputStream); - return outputStream.bytes().streamInput(); + try (BytesStreamOutput outputStream = new BytesStreamOutput()) { + discoveryNodes.writeTo(outputStream); + return outputStream.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize remote discovery nodes", e); + } } @Override public DiscoveryNodes deserialize(final InputStream inputStream) throws IOException { - return DiscoveryNodes.readFrom(new BytesStreamInput(toBytes(Streams.readFully(inputStream))), null); + try (StreamInput streamInput = new BytesStreamInput(toBytes(Streams.readFully(inputStream)))) { + return DiscoveryNodes.readFrom(streamInput, null); + } catch (IOException e) { + throw new IOException("Failed to deserialize remote discovery nodes", e); + } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java index 4ece6404e7199..1cccdd959369b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -86,14 +86,21 @@ public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { @Override public InputStream serialize() throws IOException { - BytesStreamOutput bytesStreamOutput = new BytesStreamOutput(); - hashesOfConsistentSettings.writeTo(bytesStreamOutput); - return bytesStreamOutput.bytes().streamInput(); + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { + hashesOfConsistentSettings.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize hashes of consistent settings", e); + } + } @Override public DiffableStringMap deserialize(final InputStream inputStream) throws IOException { - StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream))); - return DiffableStringMap.readFrom(in); + try (StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream)))) { + return DiffableStringMap.readFrom(in); + } catch (IOException e) { + throw new IOException("Failed to deserialize hashes of consistent settings", e); + } } } From f147333ecb4e806012df7514eabfad296c6746b5 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 06:58:41 +0530 Subject: [PATCH 05/12] Add tests Signed-off-by: Shivansh Arora --- .../org/opensearch/cluster/DiffableUtils.java | 16 ++ .../cluster/block/ClusterBlocks.java | 16 -- .../cluster/metadata/DiffableStringMap.java | 2 +- .../remote/ClusterStateDiffManifest.java | 69 ++--- .../model/ClusterStateDiffManifestTests.java | 159 ++++++++++ .../model/RemoteClusterBlocksTests.java | 13 +- .../model/RemoteDiscoveryNodesTests.java | 271 ++++++++++++++++++ ...RemoteHashesOfConsistentSettingsTests.java | 204 +++++++++++++ 8 files changed, 674 insertions(+), 76 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java index a38fc81bebc08..d21cd354bf659 100644 --- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java @@ -494,6 +494,18 @@ public void writeDiff(Diff value, StreamOutput out) throws IOException { * @opensearch.internal */ public abstract static class NonDiffableValueSerializer implements ValueSerializer { + private static final NonDiffableValueSerializer ABSTRACT_INSTANCE = new NonDiffableValueSerializer<>() { + @Override + public void write(Object value, StreamOutput out) { + throw new UnsupportedOperationException(); + } + + @Override + public Object read(StreamInput in, Object key) { + throw new UnsupportedOperationException(); + } + }; + @Override public boolean supportsDiffableValues() { return false; @@ -513,6 +525,10 @@ public void writeDiff(Diff value, StreamOutput out) throws IOException { public Diff readDiff(StreamInput in, K key) throws IOException { throw new UnsupportedOperationException(); } + + public static NonDiffableValueSerializer getAbstractInstance() { + return ABSTRACT_INSTANCE; + } } /** diff --git a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java index 377e8204e9f2f..02a20b7681ba7 100644 --- a/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java +++ b/server/src/main/java/org/opensearch/cluster/block/ClusterBlocks.java @@ -49,7 +49,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.function.Predicate; @@ -326,21 +325,6 @@ public static Diff readDiffFrom(StreamInput in) throws IOExceptio return AbstractDiffable.readDiffFrom(ClusterBlocks::readFrom, in); } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ClusterBlocks that = (ClusterBlocks) o; - return Objects.equals(global, that.global) - && Objects.equals(indicesBlocks, that.indicesBlocks) - && Objects.equals(levelHolders, that.levelHolders); - } - - @Override - public int hashCode() { - return Objects.hash(global, indicesBlocks, levelHolders); - } - /** * An immutable level holder. * diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java index a8102182576ff..5865891c8a7f9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java @@ -66,7 +66,7 @@ public static DiffableStringMap readFrom(StreamInput in) throws IOException { return map.isEmpty() ? EMPTY : new DiffableStringMap(map); } - DiffableStringMap(final Map map) { + public DiffableStringMap(final Map map) { this.innerMap = Collections.unmodifiableMap(map); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 9821f5c0b85da..de5d90f8633af 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -23,9 +23,13 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; +import static org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer.getAbstractInstance; +import static org.opensearch.cluster.DiffableUtils.getStringKeySerializer; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; /** @@ -69,30 +73,24 @@ public class ClusterStateDiffManifest implements ToXContentObject { private final List clusterStateCustomUpdated; private final List clusterStateCustomDeleted; - ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { + public ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { fromStateUUID = previousState.stateUUID(); toStateUUID = state.stateUUID(); coordinationMetadataUpdated = !Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); settingsMetadataUpdated = !Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); transientSettingsMetadataUpdate = !Metadata.isTransientSettingsMetadataEqual(state.metadata(), previousState.metadata()); templatesMetadataUpdated = !Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); - indicesDeleted = findRemovedIndices(state.metadata().indices(), previousState.metadata().indices()); - indicesUpdated = findUpdatedIndices(state.metadata().indices(), previousState.metadata().indices()); + DiffableUtils.MapDiff> indicesDiff = + DiffableUtils.diff(previousState.metadata().indices(), state.metadata().indices(), getStringKeySerializer()); + indicesDeleted = indicesDiff.getDeletes(); + indicesUpdated = new ArrayList<>(indicesDiff.getDiffs().keySet()); + indicesUpdated.addAll(indicesDiff.getUpserts().keySet()); clusterBlocksUpdated = !state.blocks().equals(previousState.blocks()); discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); - customMetadataUpdated = new ArrayList<>(); - for (String custom : state.metadata().customs().keySet()) { - if (!previousState.metadata().customs().containsKey(custom) - || !state.metadata().customs().get(custom).equals(previousState.metadata().customs().get(custom))) { - customMetadataUpdated.add(custom); - } - } - customMetadataDeleted = new ArrayList<>(); - for (String custom : previousState.metadata().customs().keySet()) { - if (state.metadata().customs().get(custom) == null) { - customMetadataDeleted.add(custom); - } - } + DiffableUtils.MapDiff> customDiff = DiffableUtils.diff(previousState.metadata().customs(), state.metadata().customs(), getStringKeySerializer(), getAbstractInstance()); + customMetadataUpdated = new ArrayList<>(customDiff.getDiffs().keySet()); + customMetadataUpdated.addAll(customDiff.getUpserts().keySet()); + customMetadataDeleted = customDiff.getDeletes(); DiffableUtils.MapDiff> routingTableDiff = RemoteRoutingTableService .getIndicesRoutingMapDiff(previousState.getRoutingTable(), state.getRoutingTable()); @@ -104,18 +102,10 @@ public class ClusterStateDiffManifest implements ToXContentObject { hashesOfConsistentSettingsUpdated = !state.metadata() .hashesOfConsistentSettings() .equals(previousState.metadata().hashesOfConsistentSettings()); - clusterStateCustomUpdated = new ArrayList<>(); - clusterStateCustomDeleted = new ArrayList<>(); - for (String custom : state.customs().keySet()) { - if (!previousState.customs().containsKey(custom) || !state.customs().get(custom).equals(previousState.customs().get(custom))) { - clusterStateCustomUpdated.add(custom); - } - } - for (String custom : previousState.customs().keySet()) { - if (state.customs().get(custom) == null) { - clusterStateCustomDeleted.add(custom); - } - } + DiffableUtils.MapDiff> clusterStateCustomDiff = DiffableUtils.diff(previousState.customs(), state.customs(), getStringKeySerializer(), getAbstractInstance()); + clusterStateCustomUpdated = new ArrayList<>(clusterStateCustomDiff.getDiffs().keySet()); + clusterStateCustomUpdated.addAll(clusterStateCustomDiff.getUpserts().keySet()); + clusterStateCustomDeleted = clusterStateCustomDiff.getDeletes(); } public ClusterStateDiffManifest( @@ -365,17 +355,6 @@ public String toString() { return Strings.toString(MediaTypeRegistry.JSON, this); } - public List findRemovedIndices(Map indices, Map previousIndices) { - List removedIndices = new ArrayList<>(); - for (String index : previousIndices.keySet()) { - // index present in previous state but not in current - if (!indices.containsKey(index)) { - removedIndices.add(index); - } - } - return removedIndices; - } - private static List convertListToString(List list) { List convertedList = new ArrayList<>(); for (Object o : list) { @@ -384,18 +363,6 @@ private static List convertListToString(List list) { return convertedList; } - public List findUpdatedIndices(Map indices, Map previousIndices) { - List updatedIndices = new ArrayList<>(); - for (String index : indices.keySet()) { - if (!previousIndices.containsKey(index)) { - updatedIndices.add(index); - } else if (previousIndices.get(index).getVersion() != indices.get(index).getVersion()) { - updatedIndices.add(index); - } - } - return updatedIndices; - } - public String getFromStateUUID() { return fromStateUUID; } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java new file mode 100644 index 0000000000000..3b445d02ddcb8 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java @@ -0,0 +1,159 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.gateway.remote.ClusterStateDiffManifest; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static org.opensearch.Version.CURRENT; +import static org.opensearch.cluster.ClusterState.EMPTY_STATE; +import static org.opensearch.core.common.transport.TransportAddress.META_ADDRESS; +import static org.opensearch.gateway.remote.model.RemoteClusterBlocksTests.randomClusterBlocks; + +public class ClusterStateDiffManifestTests extends OpenSearchTestCase { + + public void testClusterStateDiffManifest() { + ClusterState initialState = ClusterState.builder(EMPTY_STATE) + .metadata( + Metadata.builder() + .put( + IndexMetadata.builder("index-1").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + ) + ).build(); + verifyDiff( + initialState, + singletonList(IndexMetadata.builder("index-2").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build()), + singletonList("index-1"), + Collections.emptyMap(), + Collections.emptyList(), + Collections.emptyMap(), + Collections.emptyList(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + } + + private void verifyDiff( + ClusterState initialState, + List indicesToAdd, + List indicesToRemove, + Map customsToAdd, + List customsToRemove, + Map clusterStateCustomsToAdd, + List clusterStateCustomsToRemove, + boolean updateCoordinationState, + boolean updatePersistentSettings, + boolean updateTemplates, + boolean updateTransientSettings, + boolean updateDiscoveryNodes, + boolean updateClusterBlocks, + boolean updateHashesOfConsistentSettings + ) { + ClusterState.Builder clusterStateBuilder = ClusterState.builder(initialState); + Metadata.Builder metadataBuilder = Metadata.builder(initialState.metadata()); + for (IndexMetadata indexMetadata : indicesToAdd) { + metadataBuilder.put(indexMetadata, true); + } + indicesToRemove.forEach(metadataBuilder::remove); + for (String custom : customsToAdd.keySet()) { + metadataBuilder.putCustom(custom, customsToAdd.get(custom)); + } + customsToRemove.forEach(metadataBuilder::removeCustom); + for (String custom : clusterStateCustomsToAdd.keySet()) { + clusterStateBuilder.putCustom(custom, clusterStateCustomsToAdd.get(custom)); + } + clusterStateCustomsToRemove.forEach(clusterStateBuilder::removeCustom); + if (updateCoordinationState) { + metadataBuilder.coordinationMetadata( + CoordinationMetadata.builder(initialState.metadata().coordinationMetadata()) + .addVotingConfigExclusion(new CoordinationMetadata.VotingConfigExclusion("exlucdedNodeId", "excludedNodeName")) + .build() + ); + } + if (updatePersistentSettings) { + metadataBuilder.persistentSettings( + Settings.builder().put("key", "value").build() + ); + } + if (updateTemplates) { + metadataBuilder.templates(TemplatesMetadata.builder() + .put( + IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .patterns(asList("bar-*", "foo-*")) + .settings( + Settings.builder() + .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)) + .build() + ) + .build() + ).build() + ); + } + if (updateTransientSettings) { + metadataBuilder.transientSettings( + Settings.builder().put("key", "value").build() + ); + } + if (updateDiscoveryNodes) { + clusterStateBuilder.nodes(DiscoveryNodes.builder(initialState.nodes()) + .add(new DiscoveryNode("new-cluster-manager", new TransportAddress(META_ADDRESS, 9200), CURRENT)) + .clusterManagerNodeId("new-cluster-manager")); + } + if (updateHashesOfConsistentSettings) { + metadataBuilder.hashesOfConsistentSettings(Collections.singletonMap("key", "value")); + } + if (updateClusterBlocks) { + clusterStateBuilder.blocks(randomClusterBlocks()); + } + ClusterState updatedClusterState = clusterStateBuilder.metadata(metadataBuilder.build()).build(); + + ClusterStateDiffManifest manifest = new ClusterStateDiffManifest(updatedClusterState, initialState); + assertEquals(indicesToAdd.stream().map(im -> im.getIndex().getName()).collect(toList()), manifest.getIndicesUpdated()); + assertEquals(indicesToRemove, manifest.getIndicesDeleted()); + assertEquals(new ArrayList<>(customsToAdd.keySet()), manifest.getCustomMetadataUpdated()); + assertEquals(customsToRemove, manifest.getCustomMetadataDeleted()); + assertEquals(new ArrayList<>(clusterStateCustomsToAdd.keySet()), manifest.getClusterStateCustomUpdated()); + assertEquals(clusterStateCustomsToRemove, manifest.getClusterStateCustomDeleted()); + assertEquals(updateCoordinationState, manifest.isCoordinationMetadataUpdated()); + assertEquals(updatePersistentSettings, manifest.isSettingsMetadataUpdated()); + assertEquals(updateTemplates, manifest.isTemplatesMetadataUpdated()); + assertEquals(updateTransientSettings, manifest.isTransientSettingsMetadataUpdated()); + assertEquals(updateDiscoveryNodes, manifest.isDiscoveryNodesUpdated()); + assertEquals(updateClusterBlocks, manifest.isClusterBlocksUpdated()); + assertEquals(updateHashesOfConsistentSettings, manifest.isHashesOfConsistentSettingsUpdated()); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java index fffdffa0c2373..85d07ae7f5fe1 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java @@ -165,13 +165,10 @@ public void testGetUploadedMetadata() throws IOException { namedXContentRegistry ); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); - - try (InputStream inputStream = remoteObjectForUpload.serialize()) { - remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); - ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); - assertEquals(uploadedMetadata.getComponent(), CLUSTER_BLOCKS); - assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); - } + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), CLUSTER_BLOCKS); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); } public void testSerDe() throws IOException { @@ -196,7 +193,7 @@ public void testSerDe() throws IOException { } } - private ClusterBlocks randomClusterBlocks() { + static ClusterBlocks randomClusterBlocks() { ClusterBlocks.Builder builder = ClusterBlocks.builder(); int randomGlobalBlocks = randomIntBetween(0, 10); for (int i = 0; i < randomGlobalBlocks; i++) { diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java new file mode 100644 index 0000000000000..63f7e1f70ed08 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java @@ -0,0 +1,271 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.mockito.Mock; +import org.opensearch.Version; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES; + +public class RemoteDiscoveryNodesTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/discovery-nodes"; + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( + uploadedFile, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "discovery-nodes" }); + } + + public void testBlobPathParameters() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), DISCOVERY_NODES); + } + + public void testGenerateBlobFileName() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], DISCOVERY_NODES); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION)); + } + + public void testGetUploadedMetadata() throws IOException { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), DISCOVERY_NODES); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + + public void testSerDe() throws IOException { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + DiscoveryNodes readDiscoveryNodes = remoteObjectForUpload.deserialize(inputStream); + assertEquals(nodes.getSize(), readDiscoveryNodes.getSize()); + nodes.getNodes().forEach((nodeId, node) -> assertEquals(readDiscoveryNodes.get(nodeId), node)); + assertEquals(nodes.getClusterManagerNodeId(), readDiscoveryNodes.getClusterManagerNodeId()); + } + } + + public void testExceptionDuringSerialization() throws IOException { + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( + nodes, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + doThrow(new IOException("mock-exception")).when(nodes).writeTo(any()); + IOException iea = assertThrows(IOException.class, remoteObjectForUpload::serialize); + assertEquals("Failed to serialize remote discovery nodes", iea.getMessage()); + } + + public void testExceptionDuringDeserialize() throws IOException { + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + InputStream in = mock(InputStream.class); + when(in.read(any(byte[].class))).thenThrow(new IOException("mock-exception")); + String uploadedFile = "user/local/opensearch/discovery-nodes"; + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( + uploadedFile, + clusterUUID, + compressor, + namedXContentRegistry + ); + IOException ioe = assertThrows(IOException.class, () -> remoteObjectForDownload.deserialize(in)); + assertEquals("Failed to deserialize remote discovery nodes", ioe.getMessage()); + } + + private DiscoveryNodes getDiscoveryNodes() { + return DiscoveryNodes.builder() + .add( + new DiscoveryNode( + "name_" + 1, + "node_" + 1, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), + Version.CURRENT + ) + ) + .add( + new DiscoveryNode( + "name_" + 2, + "node_" + 2, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), + Version.CURRENT + ) + ) + .add( + new DiscoveryNode( + "name_" + 3, + "node_" + 3, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), + Version.CURRENT + ) + ) + .localNodeId("name_1") + .clusterManagerNodeId("name_2") + .build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java new file mode 100644 index 0000000000000..0bb8b7d5034ce --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java @@ -0,0 +1,204 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; + +public class RemoteHashesOfConsistentSettingsTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/hashes-of-consistent-settings"; + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + uploadedFile, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertArrayEquals( + remoteObjectForDownload.getBlobPathTokens(), + new String[] { "user", "local", "opensearch", "hashes-of-consistent-settings" } + ); + } + + public void testBlobPathParameters() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(GLOBAL_METADATA_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), HASHES_OF_CONSISTENT_SETTINGS); + } + + public void testGenerateBlobFileName() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], HASHES_OF_CONSISTENT_SETTINGS); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION)); + } + + public void testGetUploadedMetadata() throws IOException { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), HASHES_OF_CONSISTENT_SETTINGS); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + + public void testSerDe() throws IOException { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + DiffableStringMap readHashesOfConsistentSettings = remoteObjectForUpload.deserialize(inputStream); + assertEquals(hashesOfConsistentSettings.entrySet(), readHashesOfConsistentSettings.entrySet()); + } + } + + private DiffableStringMap getHashesOfConsistentSettings() { + Map hashesOfConsistentSettings = new HashMap<>(); + hashesOfConsistentSettings.put("secure-setting-key", "secure-setting-value"); + return new DiffableStringMap(hashesOfConsistentSettings); + } +} From 0895c405751757cd56212a64494a79a62ce537f0 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 13:33:12 +0530 Subject: [PATCH 06/12] apply spotless Signed-off-by: Shivansh Arora --- .../remote/ClusterStateDiffManifest.java | 23 +++++--- .../model/ClusterStateDiffManifestTests.java | 54 ++++++++++--------- .../model/RemoteDiscoveryNodesTests.java | 7 ++- 3 files changed, 49 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index de5d90f8633af..f7589be2b9c73 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -23,10 +23,8 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Set; import static org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer.getAbstractInstance; import static org.opensearch.cluster.DiffableUtils.getStringKeySerializer; @@ -80,14 +78,22 @@ public ClusterStateDiffManifest(ClusterState state, ClusterState previousState) settingsMetadataUpdated = !Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); transientSettingsMetadataUpdate = !Metadata.isTransientSettingsMetadataEqual(state.metadata(), previousState.metadata()); templatesMetadataUpdated = !Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); - DiffableUtils.MapDiff> indicesDiff = - DiffableUtils.diff(previousState.metadata().indices(), state.metadata().indices(), getStringKeySerializer()); + DiffableUtils.MapDiff> indicesDiff = DiffableUtils.diff( + previousState.metadata().indices(), + state.metadata().indices(), + getStringKeySerializer() + ); indicesDeleted = indicesDiff.getDeletes(); indicesUpdated = new ArrayList<>(indicesDiff.getDiffs().keySet()); indicesUpdated.addAll(indicesDiff.getUpserts().keySet()); clusterBlocksUpdated = !state.blocks().equals(previousState.blocks()); discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); - DiffableUtils.MapDiff> customDiff = DiffableUtils.diff(previousState.metadata().customs(), state.metadata().customs(), getStringKeySerializer(), getAbstractInstance()); + DiffableUtils.MapDiff> customDiff = DiffableUtils.diff( + previousState.metadata().customs(), + state.metadata().customs(), + getStringKeySerializer(), + getAbstractInstance() + ); customMetadataUpdated = new ArrayList<>(customDiff.getDiffs().keySet()); customMetadataUpdated.addAll(customDiff.getUpserts().keySet()); customMetadataDeleted = customDiff.getDeletes(); @@ -102,7 +108,12 @@ public ClusterStateDiffManifest(ClusterState state, ClusterState previousState) hashesOfConsistentSettingsUpdated = !state.metadata() .hashesOfConsistentSettings() .equals(previousState.metadata().hashesOfConsistentSettings()); - DiffableUtils.MapDiff> clusterStateCustomDiff = DiffableUtils.diff(previousState.customs(), state.customs(), getStringKeySerializer(), getAbstractInstance()); + DiffableUtils.MapDiff> clusterStateCustomDiff = DiffableUtils.diff( + previousState.customs(), + state.customs(), + getStringKeySerializer(), + getAbstractInstance() + ); clusterStateCustomUpdated = new ArrayList<>(clusterStateCustomDiff.getDiffs().keySet()); clusterStateCustomUpdated.addAll(clusterStateCustomDiff.getUpserts().keySet()); clusterStateCustomDeleted = clusterStateCustomDiff.getDeletes(); diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java index 3b445d02ddcb8..1ba4e5276487f 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java @@ -23,9 +23,7 @@ import org.opensearch.test.OpenSearchTestCase; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,14 +42,22 @@ public void testClusterStateDiffManifest() { .metadata( Metadata.builder() .put( - IndexMetadata.builder("index-1").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + IndexMetadata.builder("index-1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) .numberOfShards(1) .numberOfReplicas(0) ) - ).build(); + ) + .build(); verifyDiff( initialState, - singletonList(IndexMetadata.builder("index-2").settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build()), + singletonList( + IndexMetadata.builder("index-2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .build() + ), singletonList("index-1"), Collections.emptyMap(), Collections.emptyList(), @@ -105,33 +111,31 @@ private void verifyDiff( ); } if (updatePersistentSettings) { - metadataBuilder.persistentSettings( - Settings.builder().put("key", "value").build() - ); + metadataBuilder.persistentSettings(Settings.builder().put("key", "value").build()); } if (updateTemplates) { - metadataBuilder.templates(TemplatesMetadata.builder() - .put( - IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) - .patterns(asList("bar-*", "foo-*")) - .settings( - Settings.builder() - .put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)) - .build() - ) - .build() - ).build() + metadataBuilder.templates( + TemplatesMetadata.builder() + .put( + IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .patterns(asList("bar-*", "foo-*")) + .settings( + Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build() + ) + .build() + ) + .build() ); } if (updateTransientSettings) { - metadataBuilder.transientSettings( - Settings.builder().put("key", "value").build() - ); + metadataBuilder.transientSettings(Settings.builder().put("key", "value").build()); } if (updateDiscoveryNodes) { - clusterStateBuilder.nodes(DiscoveryNodes.builder(initialState.nodes()) - .add(new DiscoveryNode("new-cluster-manager", new TransportAddress(META_ADDRESS, 9200), CURRENT)) - .clusterManagerNodeId("new-cluster-manager")); + clusterStateBuilder.nodes( + DiscoveryNodes.builder(initialState.nodes()) + .add(new DiscoveryNode("new-cluster-manager", new TransportAddress(META_ADDRESS, 9200), CURRENT)) + .clusterManagerNodeId("new-cluster-manager") + ); } if (updateHashesOfConsistentSettings) { metadataBuilder.hashesOfConsistentSettings(Collections.singletonMap("key", "value")); diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java index 63f7e1f70ed08..f22d71075e283 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java @@ -8,7 +8,6 @@ package org.opensearch.gateway.remote.model; -import org.mockito.Mock; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.node.DiscoveryNode; @@ -36,13 +35,13 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; -import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES; public class RemoteDiscoveryNodesTests extends OpenSearchTestCase { private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; From 7d628d2eb2d85b7396abdc3974a0536d7af08704 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 15:10:42 +0530 Subject: [PATCH 07/12] Merged codec V3 with V2 Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 101 +++++----- .../remote/ClusterStateDiffManifest.java | 76 +++++++- .../RemoteClusterStateAttributesManager.java | 91 +-------- .../remote/RemoteClusterStateService.java | 2 +- .../remote/RemoteClusterStateUtils.java | 27 +-- .../remote/RemoteStateTransferException.java | 36 ++++ .../model/RemoteClusterMetadataManifest.java | 2 +- .../remote/ClusterMetadataManifestTests.java | 179 +++++++++++++++++- .../RemoteClusterStateServiceTests.java | 8 +- 9 files changed, 342 insertions(+), 180 deletions(-) create mode 100644 server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 1f2fe4343bc05..035f9c6aeb105 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -40,9 +40,9 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. - public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. - public static final int CODEC_V3 = 3; // In Codec V3, we introduce index routing-metadata, diff and other attributes as part of manifest - // required for state publication + public static final int CODEC_V2 = 2; // In Codec V2, there are separate metadata files rather than a single global metadata file, + // also we introduce index routing-metadata, diff and other attributes as part of manifest + // required for state publication private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -96,11 +96,8 @@ private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields .coordinationMetadata(coordinationMetadata(fields)) .settingMetadata(settingsMetadata(fields)) .templatesMetadata(templatesMetadata(fields)) - .customMetadataMap(customMetadata(fields)); - } - - private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields) { - return manifestV2Builder(fields).routingTableVersion(routingTableVersion(fields)) + .customMetadataMap(customMetadata(fields)) + .routingTableVersion(routingTableVersion(fields)) .indicesRouting(indicesRouting(fields)) .discoveryNodesMetadata(discoveryNodesMetadata(fields)) .clusterBlocksMetadata(clusterBlocksMetadata(fields)) @@ -228,18 +225,12 @@ private static ClusterStateDiffManifest diffManifest(Object[] fields) { fields -> manifestV2Builder(fields).build() ); - private static final ConstructingObjectParser PARSER_V3 = new ConstructingObjectParser<>( - "cluster_metadata_manifest", - fields -> manifestV3Builder(fields).build() - ); - - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V3; + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); declareParser(PARSER_V2, CODEC_V2); - declareParser(PARSER_V3, CODEC_V3); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -283,8 +274,6 @@ private static void declareParser(ConstructingObjectParser= CODEC_V3) { parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); parser.declareObjectArray( ConstructingObjectParser.constructorArg(), @@ -531,7 +520,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_15_0)) { this.codecVersion = in.readInt(); this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in); this.uploadedSettingsMetadata = new UploadedMetadataAttribute(in); @@ -566,7 +555,11 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( in.readMap(StreamInput::readString, UploadedMetadataAttribute::new) ); - this.diffManifest = null; + if (in.readBoolean()) { + this.diffManifest = new ClusterStateDiffManifest(in); + } else { + this.diffManifest = null; + } } else { if (in.getVersion().onOrAfter(Version.V_2_12_0)) { this.codecVersion = in.readInt(); @@ -641,11 +634,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws attribute.toXContent(builder, params); } builder.endObject(); - } else if (onOrAfterCodecVersion(CODEC_V1)) { - builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); - builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); - } - if (onOrAfterCodecVersion(CODEC_V3)) { builder.field(ROUTING_TABLE_VERSION_FIELD.getPreferredName(), getRoutingTableVersion()); builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { @@ -687,6 +675,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws attribute.toXContent(builder, params); } builder.endObject(); + } else if (onOrAfterCodecVersion(CODEC_V1)) { + builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); + builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } return builder; } @@ -703,7 +694,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(indices); out.writeString(previousClusterUUID); out.writeBoolean(clusterUUIDCommitted); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_15_0)) { out.writeInt(codecVersion); uploadedCoordinationMetadata.writeTo(out); uploadedSettingsMetadata.writeTo(out); @@ -737,6 +728,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } out.writeMap(uploadedClusterStateCustomMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + if (diffManifest != null) { + out.writeBoolean(true); + diffManifest.writeTo(out); + } else { + out.writeBoolean(false); + } } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); @@ -745,7 +742,6 @@ public void writeTo(StreamOutput out) throws IOException { @Override public boolean equals(Object o) { - // ToDo: update this method to contain new attributes if (this == o) { return true; } @@ -753,30 +749,32 @@ public boolean equals(Object o) { return false; } final ClusterMetadataManifest that = (ClusterMetadataManifest) o; - return Objects.equals(indices, that.indices) - && clusterTerm == that.clusterTerm - && stateVersion == that.stateVersion - && Objects.equals(clusterUUID, that.clusterUUID) - && Objects.equals(stateUUID, that.stateUUID) - && Objects.equals(opensearchVersion, that.opensearchVersion) - && Objects.equals(nodeId, that.nodeId) - && Objects.equals(committed, that.committed) - && Objects.equals(previousClusterUUID, that.previousClusterUUID) - && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) - && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) - && Objects.equals(codecVersion, that.codecVersion) - && Objects.equals(routingTableVersion, that.routingTableVersion) - && Objects.equals(indicesRouting, that.indicesRouting) - && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) - && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) - && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) - && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) - && Objects.equals(metadataVersion, that.metadataVersion) - && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) - && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) - && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) - && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) - && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); + boolean re = Objects.equals(indices, that.indices); + re = re && clusterTerm == that.clusterTerm; + re = re && stateVersion == that.stateVersion; + re = re && Objects.equals(clusterUUID, that.clusterUUID); + re = re && Objects.equals(stateUUID, that.stateUUID); + re = re && Objects.equals(opensearchVersion, that.opensearchVersion); + re = re && Objects.equals(nodeId, that.nodeId); + re = re && Objects.equals(committed, that.committed); + re = re && Objects.equals(previousClusterUUID, that.previousClusterUUID); + re = re && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted); + re = re && Objects.equals(globalMetadataFileName, that.globalMetadataFileName); + re = re && Objects.equals(codecVersion, that.codecVersion); + re = re && Objects.equals(routingTableVersion, that.routingTableVersion); + re= re && Objects.equals(indicesRouting, that.indicesRouting); + re = re && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata); + re = re && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata); + re = re && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata); + re = re && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap); + re = re && Objects.equals(metadataVersion, that.metadataVersion); + re = re && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata); + re = re && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata); + re = re && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata); + re = re && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings); + re = re && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); + re = re && Objects.equals(diffManifest, that.diffManifest); + return re; } @Override @@ -805,7 +803,8 @@ public int hashCode() { uploadedClusterBlocksMetadata, uploadedTransientSettingsMetadata, uploadedHashesOfConsistentSettings, - uploadedClusterStateCustomMap + uploadedClusterStateCustomMap, + diffManifest ); } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index f7589be2b9c73..1c3dfb2c213e5 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -15,6 +15,9 @@ import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.remote.RemoteRoutingTableService; import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; @@ -25,6 +28,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer.getAbstractInstance; import static org.opensearch.cluster.DiffableUtils.getStringKeySerializer; @@ -35,7 +39,7 @@ * * @opensearch.internal */ -public class ClusterStateDiffManifest implements ToXContentObject { +public class ClusterStateDiffManifest implements ToXContentObject, Writeable { private static final String FROM_STATE_UUID_FIELD = "from_state_uuid"; private static final String TO_STATE_UUID_FIELD = "to_state_uuid"; private static final String METADATA_DIFF_FIELD = "metadata_diff"; @@ -57,12 +61,12 @@ public class ClusterStateDiffManifest implements ToXContentObject { private final String toStateUUID; private final boolean coordinationMetadataUpdated; private final boolean settingsMetadataUpdated; - private final boolean transientSettingsMetadataUpdate; + private final boolean transientSettingsMetadataUpdated; private final boolean templatesMetadataUpdated; - private List customMetadataUpdated; - private final List customMetadataDeleted; private final List indicesUpdated; private final List indicesDeleted; + private final List customMetadataUpdated; + private final List customMetadataDeleted; private final boolean clusterBlocksUpdated; private final boolean discoveryNodesUpdated; private final List indicesRoutingUpdated; @@ -76,7 +80,7 @@ public ClusterStateDiffManifest(ClusterState state, ClusterState previousState) toStateUUID = state.stateUUID(); coordinationMetadataUpdated = !Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); settingsMetadataUpdated = !Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); - transientSettingsMetadataUpdate = !Metadata.isTransientSettingsMetadataEqual(state.metadata(), previousState.metadata()); + transientSettingsMetadataUpdated = !Metadata.isTransientSettingsMetadataEqual(state.metadata(), previousState.metadata()); templatesMetadataUpdated = !Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); DiffableUtils.MapDiff> indicesDiff = DiffableUtils.diff( previousState.metadata().indices(), @@ -142,7 +146,7 @@ public ClusterStateDiffManifest( this.toStateUUID = toStateUUID; this.coordinationMetadataUpdated = coordinationMetadataUpdated; this.settingsMetadataUpdated = settingsMetadataUpdated; - this.transientSettingsMetadataUpdate = transientSettingsMetadataUpdate; + this.transientSettingsMetadataUpdated = transientSettingsMetadataUpdate; this.templatesMetadataUpdated = templatesMetadataUpdated; this.customMetadataUpdated = customMetadataUpdated; this.customMetadataDeleted = customMetadataDeleted; @@ -157,6 +161,26 @@ public ClusterStateDiffManifest( this.clusterStateCustomDeleted = clusterStateCustomDeleted; } + public ClusterStateDiffManifest(StreamInput in) throws IOException { + this.fromStateUUID = in.readString(); + this.toStateUUID = in.readString(); + this.coordinationMetadataUpdated = in.readBoolean(); + this.settingsMetadataUpdated = in.readBoolean(); + this.transientSettingsMetadataUpdated = in.readBoolean(); + this.templatesMetadataUpdated = in.readBoolean(); + this.indicesUpdated = in.readStringList(); + this.indicesDeleted = in.readStringList(); + this.customMetadataUpdated = in.readStringList(); + this.customMetadataDeleted = in.readStringList(); + this.clusterBlocksUpdated = in.readBoolean(); + this.discoveryNodesUpdated = in.readBoolean(); + this.indicesRoutingUpdated = in.readStringList(); + this.indicesRoutingDeleted = in.readStringList(); + this.hashesOfConsistentSettingsUpdated = in.readBoolean(); + this.clusterStateCustomUpdated = in.readStringList(); + this.clusterStateCustomDeleted = in.readStringList(); + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { { @@ -166,7 +190,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws { builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); - builder.field(TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD, transientSettingsMetadataUpdate); + builder.field(TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD, transientSettingsMetadataUpdated); builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); builder.startObject(INDICES_DIFF_FIELD); builder.startArray(UPSERTS_FIELD); @@ -391,7 +415,7 @@ public boolean isSettingsMetadataUpdated() { } public boolean isTransientSettingsMetadataUpdated() { - return transientSettingsMetadataUpdate; + return transientSettingsMetadataUpdated; } public boolean isTemplatesMetadataUpdated() { @@ -442,10 +466,46 @@ public List getClusterStateCustomDeleted() { return clusterStateCustomDeleted; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterStateDiffManifest that = (ClusterStateDiffManifest) o; + return coordinationMetadataUpdated == that.coordinationMetadataUpdated && settingsMetadataUpdated == that.settingsMetadataUpdated && transientSettingsMetadataUpdated == that.transientSettingsMetadataUpdated && templatesMetadataUpdated == that.templatesMetadataUpdated && clusterBlocksUpdated == that.clusterBlocksUpdated && discoveryNodesUpdated == that.discoveryNodesUpdated && hashesOfConsistentSettingsUpdated == that.hashesOfConsistentSettingsUpdated && Objects.equals(fromStateUUID, that.fromStateUUID) && Objects.equals(toStateUUID, that.toStateUUID) && Objects.equals(customMetadataUpdated, that.customMetadataUpdated) && Objects.equals(customMetadataDeleted, that.customMetadataDeleted) && Objects.equals(indicesUpdated, that.indicesUpdated) && Objects.equals(indicesDeleted, that.indicesDeleted) && Objects.equals(indicesRoutingUpdated, that.indicesRoutingUpdated) && Objects.equals(indicesRoutingDeleted, that.indicesRoutingDeleted) && Objects.equals(clusterStateCustomUpdated, that.clusterStateCustomUpdated) && Objects.equals(clusterStateCustomDeleted, that.clusterStateCustomDeleted); + } + + @Override + public int hashCode() { + return Objects.hash(fromStateUUID, toStateUUID, coordinationMetadataUpdated, settingsMetadataUpdated, transientSettingsMetadataUpdated, templatesMetadataUpdated, customMetadataUpdated, customMetadataDeleted, indicesUpdated, indicesDeleted, clusterBlocksUpdated, discoveryNodesUpdated, indicesRoutingUpdated, indicesRoutingDeleted, hashesOfConsistentSettingsUpdated, clusterStateCustomUpdated, clusterStateCustomDeleted); + } + public static Builder builder() { return new Builder(); } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(fromStateUUID); + out.writeString(toStateUUID); + out.writeBoolean(coordinationMetadataUpdated); + out.writeBoolean(settingsMetadataUpdated); + out.writeBoolean(transientSettingsMetadataUpdated); + out.writeBoolean(templatesMetadataUpdated); + out.writeStringCollection(indicesUpdated); + out.writeStringCollection(indicesDeleted); + out.writeStringCollection(customMetadataUpdated); + out.writeStringCollection(customMetadataDeleted); + out.writeStringCollection(indicesRoutingUpdated); + out.writeStringCollection(indicesRoutingDeleted); + out.writeBoolean(clusterBlocksUpdated); + out.writeBoolean(discoveryNodesUpdated); + out.writeStringCollection(indicesRoutingUpdated); + out.writeStringCollection(indicesRoutingDeleted); + out.writeBoolean(hashesOfConsistentSettingsUpdated); + out.writeStringCollection(clusterStateCustomUpdated); + out.writeStringCollection(clusterStateCustomDeleted); + } + /** * Builder for ClusterStateDiffManifest * diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index 100a26b3bdc5a..d9b225ed19be9 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -20,7 +20,6 @@ import org.opensearch.core.compress.Compressor; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.gateway.remote.RemoteClusterStateUtils.RemoteStateTransferException; import org.opensearch.gateway.remote.model.RemoteClusterBlocks; import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms; @@ -73,46 +72,15 @@ public class RemoteClusterStateAttributesManager { * Allows async upload of Cluster State Attribute components to remote */ CheckedRunnable getAsyncMetadataWriteAction( - ClusterState clusterState, String component, - ToXContent componentData, + AbstractRemoteWritableBlobEntity blobEntity, + RemoteClusterStateBlobStore remoteEntityStore, LatchedActionListener latchedActionListener ) { - if (componentData instanceof DiscoveryNodes) { - RemoteDiscoveryNodes remoteObject = new RemoteDiscoveryNodes( - (DiscoveryNodes) componentData, - clusterState.version(), - clusterState.metadata().clusterUUID(), - compressor, - namedXContentRegistry - ); - return () -> discoveryNodesBlobStore.writeAsync( - remoteObject, - getActionListener(component, remoteObject, latchedActionListener) - ); - } else if (componentData instanceof ClusterBlocks) { - RemoteClusterBlocks remoteObject = new RemoteClusterBlocks( - (ClusterBlocks) componentData, - clusterState.version(), - clusterState.metadata().clusterUUID(), - compressor, - namedXContentRegistry - ); - return () -> clusterBlocksBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); - } else if (componentData instanceof ClusterState.Custom) { - RemoteClusterStateCustoms remoteObject = new RemoteClusterStateCustoms( - (ClusterState.Custom) componentData, - component, - clusterState.version(), - clusterState.metadata().clusterUUID(), - compressor, - namedXContentRegistry, - namedWriteableRegistry - ); - return () -> customsBlobStore.writeAsync(remoteObject, getActionListener(component, remoteObject, latchedActionListener)); - } else { - throw new RemoteStateTransferException("Remote object not found for " + componentData.getClass()); - } + return () -> remoteEntityStore.writeAsync( + blobEntity, + getActionListener(component, blobEntity, latchedActionListener) + ); } private ActionListener getActionListener( @@ -122,60 +90,21 @@ private ActionListener getActionListener( ) { return ActionListener.wrap( resp -> latchedActionListener.onResponse(remoteObject.getUploadedMetadata()), - ex -> latchedActionListener.onFailure(new RemoteClusterStateUtils.RemoteStateTransferException(component, remoteObject, ex)) + ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, remoteObject, ex)) ); } public CheckedRunnable getAsyncMetadataReadAction( - String clusterUUID, String component, - String componentName, - String uploadedFilename, + AbstractRemoteWritableBlobEntity blobEntity, + RemoteClusterStateBlobStore remoteEntityStore, LatchedActionListener listener ) { final ActionListener actionListener = ActionListener.wrap( response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), listener::onFailure ); - if (component.equals(RemoteDiscoveryNodes.DISCOVERY_NODES)) { - RemoteDiscoveryNodes remoteDiscoveryNodes = new RemoteDiscoveryNodes( - uploadedFilename, - clusterUUID, - compressor, - namedXContentRegistry - ); - return () -> discoveryNodesBlobStore.readAsync(remoteDiscoveryNodes, actionListener); - } else if (component.equals(RemoteClusterBlocks.CLUSTER_BLOCKS)) { - RemoteClusterBlocks remoteClusterBlocks = new RemoteClusterBlocks( - uploadedFilename, - clusterUUID, - compressor, - namedXContentRegistry - ); - return () -> clusterBlocksBlobStore.readAsync(remoteClusterBlocks, actionListener); - } else if (component.equals(CLUSTER_STATE_CUSTOM)) { - final ActionListener customActionListener = ActionListener.wrap( - response -> listener.onResponse( - new RemoteReadResult( - (ToXContent) response, - CLUSTER_STATE_ATTRIBUTE, - String.join(CUSTOM_DELIMITER, component, componentName) - ) - ), - listener::onFailure - ); - RemoteClusterStateCustoms remoteClusterStateCustoms = new RemoteClusterStateCustoms( - uploadedFilename, - componentName, - clusterUUID, - compressor, - namedXContentRegistry, - namedWriteableRegistry - ); - return () -> customsBlobStore.readAsync(remoteClusterStateCustoms, customActionListener); - } else { - throw new RemoteStateTransferException("Remote object not found for " + component); - } + return () -> remoteEntityStore.readAsync(blobEntity, actionListener); } public Map getUpdatedCustoms(ClusterState clusterState, ClusterState previousClusterState) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 50487315d5867..59612666806b7 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -70,6 +70,7 @@ import static java.util.Objects.requireNonNull; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; @@ -216,7 +217,6 @@ public class RemoteClusterStateService implements Closeable { + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; // ToXContent Params with gateway mode. diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 4e260725648b4..25b3c53a1654f 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -53,7 +53,7 @@ public static String encodeString(String content) { } public static String getFormattedFileName(String fileName, int codecVersion) { - if (codecVersion < ClusterMetadataManifest.CODEC_V3) { + if (codecVersion < ClusterMetadataManifest.CODEC_V2) { return String.format(Locale.ROOT, METADATA_NAME_FORMAT, fileName); } return fileName; @@ -68,31 +68,6 @@ static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepositor ); } - /** - * Exception for Remote state transfer. - */ - public static class RemoteStateTransferException extends RuntimeException { - private RemoteWriteableEntity entity; - - public RemoteStateTransferException(String errorDesc) { - super(errorDesc); - } - - public RemoteStateTransferException(String errorDesc, Throwable cause) { - super(errorDesc, cause); - } - - public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity) { - super(errorDesc); - this.entity = entity; - } - - public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity, Throwable cause) { - super(errorDesc, cause); - this.entity = entity; - } - } - /** * Container class to keep metadata of all uploaded attributes */ diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java new file mode 100644 index 0000000000000..fef3e734e78eb --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java @@ -0,0 +1,36 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.common.remote.RemoteWriteableEntity; + +/** + * Exception for Remote state transfer. + */ +public class RemoteStateTransferException extends RuntimeException { + private RemoteWriteableEntity entity; + + public RemoteStateTransferException(String errorDesc) { + super(errorDesc); + } + + public RemoteStateTransferException(String errorDesc, Throwable cause) { + super(errorDesc, cause); + } + + public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity) { + super(errorDesc); + this.entity = entity; + } + + public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity, Throwable cause) { + super(errorDesc, cause); + this.entity = entity; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index 62bc4ce022a37..f7024c8fde7ed 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -35,7 +35,7 @@ public class RemoteClusterMetadataManifest extends AbstractRemoteWritableBlobEnt public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; public static final String COMMITTED = "C"; public static final String PUBLISHED = "P"; diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index e868004f90d60..43739c0ba72a0 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -9,6 +9,7 @@ package org.opensearch.gateway.remote; import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; @@ -33,7 +34,11 @@ import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; +import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; +import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; public class ClusterMetadataManifestTests extends OpenSearchTestCase { @@ -100,7 +105,7 @@ public void testClusterMetadataManifestXContent() throws IOException { .opensearchVersion(Version.CURRENT) .nodeId("test-node-id") .committed(false) - .codecVersion(ClusterMetadataManifest.CODEC_V3) + .codecVersion(ClusterMetadataManifest.CODEC_V2) .indices(Collections.singletonList(uploadedIndexMetadata)) .previousClusterUUID("prev-cluster-uuid") .clusterUUIDCommitted(true) @@ -149,7 +154,7 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { .opensearchVersion(Version.CURRENT) .nodeId("B10RX1f5RJenMQvYccCgSQ") .committed(true) - .codecVersion(ClusterMetadataManifest.CODEC_V3) + .codecVersion(ClusterMetadataManifest.CODEC_V2) .indices(randomUploadedIndexMetadataList()) .previousClusterUUID("yfObdx8KSMKKrXf8UyHhM") .clusterUUIDCommitted(true) @@ -178,7 +183,11 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { ) .routingTableVersion(1L) .discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, "discovery-nodes-file")) - + .clusterBlocksMetadata(new UploadedMetadataAttribute(CLUSTER_BLOCKS, "cluster-block-file")) + .transientSettingsMetadata(new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, "transient-settings-file")) + .hashesOfConsistentSettings(new UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, "hashes-of-consistent-settings-file")) + .clusterStateCustomMetadataMap(Collections.emptyMap()) + .diffManifest(new ClusterStateDiffManifest(generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE)) .build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( @@ -316,6 +325,150 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { } ); } + { + // Mutate Coordination metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.coordinationMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate setting metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.settingMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate template metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.templatesMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate custom metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.customMetadataMap(Collections.emptyMap()); + return builder.build(); + } + ); + } + { + // Mutate discovery nodes + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.discoveryNodesMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate cluster blocks + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.clusterBlocksMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate transient settings metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.transientSettingsMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate diff manifest + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.diffManifest(null); + return builder.build(); + } + ); + } + { + // Mutate hashes of consistent settings + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.hashesOfConsistentSettings(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } } public void testClusterMetadataManifestXContentV2() throws IOException { @@ -356,6 +509,14 @@ public void testClusterMetadataManifestXContentV2() throws IOException { ) ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) ) + .routingTableVersion(1L) + .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) + .discoveryNodesMetadata(uploadedMetadataAttribute) + .clusterBlocksMetadata(uploadedMetadataAttribute) + .transientSettingsMetadata(uploadedMetadataAttribute) + .hashesOfConsistentSettings(uploadedMetadataAttribute) + .clusterStateCustomMetadataMap(Collections.emptyMap()) + .diffManifest(new ClusterStateDiffManifest(generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE)) .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -363,12 +524,12 @@ public void testClusterMetadataManifestXContentV2() throws IOException { builder.endObject(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV2(parser); + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); assertEquals(originalManifest, fromXContentManifest); } } - public void testClusterMetadataManifestXContentV3() throws IOException { + public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() @@ -379,7 +540,7 @@ public void testClusterMetadataManifestXContentV3() throws IOException { .opensearchVersion(Version.CURRENT) .nodeId("test-node-id") .committed(false) - .codecVersion(ClusterMetadataManifest.CODEC_V3) + .codecVersion(ClusterMetadataManifest.CODEC_V2) .indices(Collections.singletonList(uploadedIndexMetadata)) .previousClusterUUID("prev-cluster-uuid") .clusterUUIDCommitted(true) @@ -406,8 +567,6 @@ public void testClusterMetadataManifestXContentV3() throws IOException { ) ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) ) - .routingTableVersion(1L) - .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); @@ -433,6 +592,10 @@ private UploadedIndexMetadata randomUploadedIndexMetadata() { return new UploadedIndexMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10)); } + private UploadedMetadataAttribute randomUploadedMetadataAttribute() { + return new UploadedMetadataAttribute("attribute_name", "testing_attribute"); + } + public void testUploadedIndexMetadataSerializationEqualsHashCode() { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); EqualsHashCodeTestUtils.checkEqualsAndHashCode( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 4a53770c76d88..1c06b50da910d 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -90,11 +90,11 @@ import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -1471,7 +1471,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2); mockBlobContainerForGlobalMetadata(blobContainer1, clusterManifest1, metadata1); - mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V3); + mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V2); List uploadedIndexMetadataList2 = List.of( new UploadedIndexMetadata("index1", "index-uuid1", "key1"), @@ -1503,7 +1503,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4); mockBlobContainerForGlobalMetadata(blobContainer2, clusterManifest2, metadata2); - mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V3); + mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V2); // differGlobalMetadata controls which one of IndexMetadata or Metadata object would be different // when comparing cluster-uuid3 and cluster-uuid1 state. @@ -1537,7 +1537,7 @@ private void mockObjectsForGettingPreviousClusterUUID( clusterUUIDCommitted.getOrDefault("cluster-uuid3", true) ); mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3); - mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V3); + mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V2); ArrayList mockBlobContainerOrderedList = new ArrayList<>( List.of(blobContainer1, blobContainer1, blobContainer3, blobContainer3, blobContainer2, blobContainer2) From fc6fb72781e547a87959d72650581321be9ad2e2 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 17:36:15 +0530 Subject: [PATCH 08/12] apply spotless Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 2 +- .../remote/ClusterStateDiffManifest.java | 38 ++++++++++++++++++- .../RemoteClusterStateAttributesManager.java | 8 +--- .../remote/RemoteClusterStateUtils.java | 1 - 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 035f9c6aeb105..243819c2ed99e 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -762,7 +762,7 @@ public boolean equals(Object o) { re = re && Objects.equals(globalMetadataFileName, that.globalMetadataFileName); re = re && Objects.equals(codecVersion, that.codecVersion); re = re && Objects.equals(routingTableVersion, that.routingTableVersion); - re= re && Objects.equals(indicesRouting, that.indicesRouting); + re = re && Objects.equals(indicesRouting, that.indicesRouting); re = re && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata); re = re && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata); re = re && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata); diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 1c3dfb2c213e5..41849545f67ec 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -471,12 +471,46 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ClusterStateDiffManifest that = (ClusterStateDiffManifest) o; - return coordinationMetadataUpdated == that.coordinationMetadataUpdated && settingsMetadataUpdated == that.settingsMetadataUpdated && transientSettingsMetadataUpdated == that.transientSettingsMetadataUpdated && templatesMetadataUpdated == that.templatesMetadataUpdated && clusterBlocksUpdated == that.clusterBlocksUpdated && discoveryNodesUpdated == that.discoveryNodesUpdated && hashesOfConsistentSettingsUpdated == that.hashesOfConsistentSettingsUpdated && Objects.equals(fromStateUUID, that.fromStateUUID) && Objects.equals(toStateUUID, that.toStateUUID) && Objects.equals(customMetadataUpdated, that.customMetadataUpdated) && Objects.equals(customMetadataDeleted, that.customMetadataDeleted) && Objects.equals(indicesUpdated, that.indicesUpdated) && Objects.equals(indicesDeleted, that.indicesDeleted) && Objects.equals(indicesRoutingUpdated, that.indicesRoutingUpdated) && Objects.equals(indicesRoutingDeleted, that.indicesRoutingDeleted) && Objects.equals(clusterStateCustomUpdated, that.clusterStateCustomUpdated) && Objects.equals(clusterStateCustomDeleted, that.clusterStateCustomDeleted); + return coordinationMetadataUpdated == that.coordinationMetadataUpdated + && settingsMetadataUpdated == that.settingsMetadataUpdated + && transientSettingsMetadataUpdated == that.transientSettingsMetadataUpdated + && templatesMetadataUpdated == that.templatesMetadataUpdated + && clusterBlocksUpdated == that.clusterBlocksUpdated + && discoveryNodesUpdated == that.discoveryNodesUpdated + && hashesOfConsistentSettingsUpdated == that.hashesOfConsistentSettingsUpdated + && Objects.equals(fromStateUUID, that.fromStateUUID) + && Objects.equals(toStateUUID, that.toStateUUID) + && Objects.equals(customMetadataUpdated, that.customMetadataUpdated) + && Objects.equals(customMetadataDeleted, that.customMetadataDeleted) + && Objects.equals(indicesUpdated, that.indicesUpdated) + && Objects.equals(indicesDeleted, that.indicesDeleted) + && Objects.equals(indicesRoutingUpdated, that.indicesRoutingUpdated) + && Objects.equals(indicesRoutingDeleted, that.indicesRoutingDeleted) + && Objects.equals(clusterStateCustomUpdated, that.clusterStateCustomUpdated) + && Objects.equals(clusterStateCustomDeleted, that.clusterStateCustomDeleted); } @Override public int hashCode() { - return Objects.hash(fromStateUUID, toStateUUID, coordinationMetadataUpdated, settingsMetadataUpdated, transientSettingsMetadataUpdated, templatesMetadataUpdated, customMetadataUpdated, customMetadataDeleted, indicesUpdated, indicesDeleted, clusterBlocksUpdated, discoveryNodesUpdated, indicesRoutingUpdated, indicesRoutingDeleted, hashesOfConsistentSettingsUpdated, clusterStateCustomUpdated, clusterStateCustomDeleted); + return Objects.hash( + fromStateUUID, + toStateUUID, + coordinationMetadataUpdated, + settingsMetadataUpdated, + transientSettingsMetadataUpdated, + templatesMetadataUpdated, + customMetadataUpdated, + customMetadataDeleted, + indicesUpdated, + indicesDeleted, + clusterBlocksUpdated, + discoveryNodesUpdated, + indicesRoutingUpdated, + indicesRoutingDeleted, + hashesOfConsistentSettingsUpdated, + clusterStateCustomUpdated, + clusterStateCustomDeleted + ); } public static Builder builder() { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java index d9b225ed19be9..7e83a7bf7da44 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -32,9 +32,6 @@ import java.util.Map; import java.util.Set; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; -import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; - /** * A Manager which provides APIs to upload and download attributes of ClusterState to the {@link RemoteClusterStateBlobStore} * @@ -77,10 +74,7 @@ CheckedRunnable getAsyncMetadataWriteAction( RemoteClusterStateBlobStore remoteEntityStore, LatchedActionListener latchedActionListener ) { - return () -> remoteEntityStore.writeAsync( - blobEntity, - getActionListener(component, blobEntity, latchedActionListener) - ); + return () -> remoteEntityStore.writeAsync(blobEntity, getActionListener(component, blobEntity, latchedActionListener)); } private ActionListener getActionListener( diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index 25b3c53a1654f..8efde7ee45f49 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -11,7 +11,6 @@ import org.opensearch.cluster.metadata.Metadata; import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobPath; -import org.opensearch.common.remote.RemoteWriteableEntity; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.repositories.blobstore.BlobStoreRepository; From 3868cb9398cfd3c105b4fd93732ef43e030d1b40 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 17:58:50 +0530 Subject: [PATCH 09/12] revert version change Signed-off-by: Shivansh Arora --- .../opensearch/gateway/remote/ClusterMetadataManifest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 243819c2ed99e..961067b691020 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -520,7 +520,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); this.previousClusterUUID = in.readString(); this.clusterUUIDCommitted = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_2_15_0)) { + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { this.codecVersion = in.readInt(); this.uploadedCoordinationMetadata = new UploadedMetadataAttribute(in); this.uploadedSettingsMetadata = new UploadedMetadataAttribute(in); @@ -694,7 +694,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeCollection(indices); out.writeString(previousClusterUUID); out.writeBoolean(clusterUUIDCommitted); - if (out.getVersion().onOrAfter(Version.V_2_15_0)) { + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { out.writeInt(codecVersion); uploadedCoordinationMetadata.writeTo(out); uploadedSettingsMetadata.writeTo(out); From f39d68684bf59bcbfb16c477cf28f8bd496c5d96 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 18:48:43 +0530 Subject: [PATCH 10/12] Address PR comments Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 4 - .../remote/ClusterStateDiffManifest.java | 91 +++++++++-------- .../remote/RemoteStateTransferException.java | 9 ++ .../remote/model/RemoteClusterBlocks.java | 20 +--- .../model/RemoteClusterStateCustoms.java | 10 +- .../remote/model/RemoteDiscoveryNodes.java | 15 +-- .../RemoteHashesOfConsistentSettings.java | 15 +-- .../model/ClusterStateDiffManifestTests.java | 59 +++++++++-- .../model/RemoteClusterBlocksTests.java | 79 +++------------ .../model/RemoteDiscoveryNodesTests.java | 99 +++---------------- ...RemoteHashesOfConsistentSettingsTests.java | 33 +++---- 11 files changed, 157 insertions(+), 277 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 961067b691020..b6323279e0cd2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -424,10 +424,6 @@ public ClusterStateDiffManifest getDiffManifest() { return diffManifest; } - public UploadedMetadataAttribute getDiscoverNodeMetadata() { - return uploadedDiscoveryNodesMetadata; - } - public Map getClusterStateCustomMap() { return uploadedClusterStateCustomMap; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index 41849545f67ec..9d950ee67c4a4 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -19,7 +19,7 @@ import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.MediaTypeRegistry; -import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParseException; import org.opensearch.core.xcontent.XContentParser; @@ -39,7 +39,7 @@ * * @opensearch.internal */ -public class ClusterStateDiffManifest implements ToXContentObject, Writeable { +public class ClusterStateDiffManifest implements ToXContentFragment, Writeable { private static final String FROM_STATE_UUID_FIELD = "from_state_uuid"; private static final String TO_STATE_UUID_FIELD = "to_state_uuid"; private static final String METADATA_DIFF_FIELD = "metadata_diff"; @@ -183,71 +183,68 @@ public ClusterStateDiffManifest(StreamInput in) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(FROM_STATE_UUID_FIELD, fromStateUUID); + builder.field(TO_STATE_UUID_FIELD, toStateUUID); + builder.startObject(METADATA_DIFF_FIELD); { - builder.field(FROM_STATE_UUID_FIELD, fromStateUUID); - builder.field(TO_STATE_UUID_FIELD, toStateUUID); - builder.startObject(METADATA_DIFF_FIELD); - { - builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); - builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); - builder.field(TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD, transientSettingsMetadataUpdated); - builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); - builder.startObject(INDICES_DIFF_FIELD); - builder.startArray(UPSERTS_FIELD); - for (String index : indicesUpdated) { - builder.value(index); - } - builder.endArray(); - builder.startArray(DELETES_FIELD); - for (String index : indicesDeleted) { - builder.value(index); - } - builder.endArray(); - builder.endObject(); - builder.startObject(METADATA_CUSTOM_DIFF_FIELD); - builder.startArray(UPSERTS_FIELD); - for (String custom : customMetadataUpdated) { - builder.value(custom); - } - builder.endArray(); - builder.startArray(DELETES_FIELD); - for (String custom : customMetadataDeleted) { - builder.value(custom); - } - builder.endArray(); - builder.endObject(); - builder.field(HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD, hashesOfConsistentSettingsUpdated); - } - builder.endObject(); - builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); - builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); - - builder.startObject(ROUTING_TABLE_DIFF); + builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); + builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); + builder.field(TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD, transientSettingsMetadataUpdated); + builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); + builder.startObject(INDICES_DIFF_FIELD); builder.startArray(UPSERTS_FIELD); - for (String index : indicesRoutingUpdated) { + for (String index : indicesUpdated) { builder.value(index); } builder.endArray(); builder.startArray(DELETES_FIELD); - for (String index : indicesRoutingDeleted) { + for (String index : indicesDeleted) { builder.value(index); } builder.endArray(); builder.endObject(); - builder.startObject(CLUSTER_STATE_CUSTOM_DIFF_FIELD); + builder.startObject(METADATA_CUSTOM_DIFF_FIELD); builder.startArray(UPSERTS_FIELD); - for (String custom : clusterStateCustomUpdated) { + for (String custom : customMetadataUpdated) { builder.value(custom); } builder.endArray(); builder.startArray(DELETES_FIELD); - for (String custom : clusterStateCustomDeleted) { + for (String custom : customMetadataDeleted) { builder.value(custom); } builder.endArray(); builder.endObject(); - + builder.field(HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD, hashesOfConsistentSettingsUpdated); + } + builder.endObject(); + builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); + builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); + + builder.startObject(ROUTING_TABLE_DIFF); + builder.startArray(UPSERTS_FIELD); + for (String index : indicesRoutingUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String index : indicesRoutingDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + builder.startObject(CLUSTER_STATE_CUSTOM_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String custom : clusterStateCustomUpdated) { + builder.value(custom); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String custom : clusterStateCustomDeleted) { + builder.value(custom); } + builder.endArray(); + builder.endObject(); return builder; } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java index fef3e734e78eb..5b75b6c18ee5b 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java @@ -33,4 +33,13 @@ public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity e super(errorDesc, cause); this.entity = entity; } + + @Override + public String toString() { + String message = super.toString(); + if (entity != null) { + message += ", failed entity:" + entity; + } + return message; + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java index 9bb0eac5bd793..937f9dc2c8631 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -16,7 +16,6 @@ import org.opensearch.core.common.io.stream.BytesStreamInput; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.compress.Compressor; -import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; import org.opensearch.index.remote.RemoteStoreUtils; @@ -40,25 +39,14 @@ public class RemoteClusterBlocks extends AbstractRemoteWritableBlobEntity indicesToAdd, List indicesToRemove, @@ -159,5 +205,6 @@ private void verifyDiff( assertEquals(updateDiscoveryNodes, manifest.isDiscoveryNodesUpdated()); assertEquals(updateClusterBlocks, manifest.isClusterBlocksUpdated()); assertEquals(updateHashesOfConsistentSettings, manifest.isHashesOfConsistentSettingsUpdated()); + return manifest; } } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java index 85d07ae7f5fe1..a5419a8cc8115 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java @@ -59,79 +59,40 @@ public void setup() { public void testClusterUUID() { ClusterBlocks clusterBlocks = randomClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( - clusterBlocks, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( - TEST_BLOB_NAME, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); } public void testFullBlobName() { ClusterBlocks clusterBlocks = randomClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( - clusterBlocks, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); assertNull(remoteObjectForUpload.getFullBlobName()); - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( - TEST_BLOB_NAME, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); } public void testBlobFileName() { ClusterBlocks clusterBlocks = randomClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( - clusterBlocks, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); assertNull(remoteObjectForUpload.getBlobFileName()); - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( - TEST_BLOB_NAME, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); } public void testBlobPathTokens() { String uploadedFile = "user/local/opensearch/cluster-blocks"; - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(uploadedFile, clusterUUID, compressor, namedXContentRegistry); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(uploadedFile, clusterUUID, compressor); assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "cluster-blocks" }); } public void testBlobPathParameters() { ClusterBlocks clusterBlocks = randomClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( - clusterBlocks, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertEquals(params.getPathTokens(), List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN)); assertEquals(params.getFilePrefix(), CLUSTER_BLOCKS); @@ -139,13 +100,7 @@ public void testBlobPathParameters() { public void testGenerateBlobFileName() { ClusterBlocks clusterBlocks = randomClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( - clusterBlocks, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); assertEquals(nameTokens[0], CLUSTER_BLOCKS); @@ -157,13 +112,7 @@ public void testGenerateBlobFileName() { public void testGetUploadedMetadata() throws IOException { ClusterBlocks clusterBlocks = randomClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( - clusterBlocks, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); @@ -173,13 +122,7 @@ public void testGetUploadedMetadata() throws IOException { public void testSerDe() throws IOException { ClusterBlocks clusterBlocks = randomClusterBlocks(); - RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks( - clusterBlocks, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); try (InputStream inputStream = remoteObjectForUpload.serialize()) { remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); assertTrue(inputStream.available() > 0); diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java index f22d71075e283..0c46960938798 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java @@ -67,84 +67,40 @@ public void setup() { public void testClusterUUID() { DiscoveryNodes nodes = getDiscoveryNodes(); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); - RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( - TEST_BLOB_NAME, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(TEST_BLOB_NAME, clusterUUID, compressor); assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); } public void testFullBlobName() { DiscoveryNodes nodes = getDiscoveryNodes(); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); assertNull(remoteObjectForUpload.getFullBlobName()); - RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( - TEST_BLOB_NAME, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(TEST_BLOB_NAME, clusterUUID, compressor); assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); } public void testBlobFileName() { DiscoveryNodes nodes = getDiscoveryNodes(); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); assertNull(remoteObjectForUpload.getBlobFileName()); - RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks( - TEST_BLOB_NAME, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); } public void testBlobPathTokens() { String uploadedFile = "user/local/opensearch/discovery-nodes"; - RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( - uploadedFile, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(uploadedFile, clusterUUID, compressor); assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "discovery-nodes" }); } public void testBlobPathParameters() { DiscoveryNodes nodes = getDiscoveryNodes(); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertEquals(params.getPathTokens(), List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN)); assertEquals(params.getFilePrefix(), DISCOVERY_NODES); @@ -152,13 +108,7 @@ public void testBlobPathParameters() { public void testGenerateBlobFileName() { DiscoveryNodes nodes = getDiscoveryNodes(); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); assertEquals(nameTokens[0], DISCOVERY_NODES); @@ -169,13 +119,7 @@ public void testGenerateBlobFileName() { public void testGetUploadedMetadata() throws IOException { DiscoveryNodes nodes = getDiscoveryNodes(); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); @@ -185,13 +129,7 @@ public void testGetUploadedMetadata() throws IOException { public void testSerDe() throws IOException { DiscoveryNodes nodes = getDiscoveryNodes(); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); try (InputStream inputStream = remoteObjectForUpload.serialize()) { remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); assertTrue(inputStream.available() > 0); @@ -204,13 +142,7 @@ public void testSerDe() throws IOException { public void testExceptionDuringSerialization() throws IOException { DiscoveryNodes nodes = mock(DiscoveryNodes.class); - RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes( - nodes, - METADATA_VERSION, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); doThrow(new IOException("mock-exception")).when(nodes).writeTo(any()); IOException iea = assertThrows(IOException.class, remoteObjectForUpload::serialize); assertEquals("Failed to serialize remote discovery nodes", iea.getMessage()); @@ -221,12 +153,7 @@ public void testExceptionDuringDeserialize() throws IOException { InputStream in = mock(InputStream.class); when(in.read(any(byte[].class))).thenThrow(new IOException("mock-exception")); String uploadedFile = "user/local/opensearch/discovery-nodes"; - RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes( - uploadedFile, - clusterUUID, - compressor, - namedXContentRegistry - ); + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(uploadedFile, clusterUUID, compressor); IOException ioe = assertThrows(IOException.class, () -> remoteObjectForDownload.deserialize(in)); assertEquals("Failed to deserialize remote discovery nodes", ioe.getMessage()); } diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java index 0bb8b7d5034ce..d883eabf9fbc9 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java @@ -64,16 +64,14 @@ public void testClusterUUID() { hashesOfConsistentSettings, METADATA_VERSION, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( TEST_BLOB_NAME, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); } @@ -84,16 +82,14 @@ public void testFullBlobName() { hashesOfConsistentSettings, METADATA_VERSION, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertNull(remoteObjectForUpload.getFullBlobName()); RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( TEST_BLOB_NAME, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); } @@ -104,16 +100,14 @@ public void testBlobFileName() { hashesOfConsistentSettings, METADATA_VERSION, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertNull(remoteObjectForUpload.getBlobFileName()); RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( TEST_BLOB_NAME, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); } @@ -123,8 +117,7 @@ public void testBlobPathTokens() { RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( uploadedFile, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertArrayEquals( remoteObjectForDownload.getBlobPathTokens(), @@ -138,8 +131,7 @@ public void testBlobPathParameters() { hashesOfConsistentSettings, METADATA_VERSION, clusterUUID, - compressor, - namedXContentRegistry + compressor ); BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); assertEquals(params.getPathTokens(), List.of(GLOBAL_METADATA_PATH_TOKEN)); @@ -152,8 +144,7 @@ public void testGenerateBlobFileName() { hashesOfConsistentSettings, METADATA_VERSION, clusterUUID, - compressor, - namedXContentRegistry + compressor ); String blobFileName = remoteObjectForUpload.generateBlobFileName(); String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); @@ -169,8 +160,7 @@ public void testGetUploadedMetadata() throws IOException { hashesOfConsistentSettings, METADATA_VERSION, clusterUUID, - compressor, - namedXContentRegistry + compressor ); assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); @@ -185,8 +175,7 @@ public void testSerDe() throws IOException { hashesOfConsistentSettings, METADATA_VERSION, clusterUUID, - compressor, - namedXContentRegistry + compressor ); try (InputStream inputStream = remoteObjectForUpload.serialize()) { remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); From 2ab8c9ed0c5c82e5cc809c1df29f26069a69be42 Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 19:24:38 +0530 Subject: [PATCH 11/12] Add UTs for RemoteClusterStateCustom Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 51 ++-- .../model/RemoteClusterStateCustomsTests.java | 259 ++++++++++++++++++ 2 files changed, 284 insertions(+), 26 deletions(-) create mode 100644 server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index b6323279e0cd2..44c08ce6a09dc 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -745,32 +745,31 @@ public boolean equals(Object o) { return false; } final ClusterMetadataManifest that = (ClusterMetadataManifest) o; - boolean re = Objects.equals(indices, that.indices); - re = re && clusterTerm == that.clusterTerm; - re = re && stateVersion == that.stateVersion; - re = re && Objects.equals(clusterUUID, that.clusterUUID); - re = re && Objects.equals(stateUUID, that.stateUUID); - re = re && Objects.equals(opensearchVersion, that.opensearchVersion); - re = re && Objects.equals(nodeId, that.nodeId); - re = re && Objects.equals(committed, that.committed); - re = re && Objects.equals(previousClusterUUID, that.previousClusterUUID); - re = re && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted); - re = re && Objects.equals(globalMetadataFileName, that.globalMetadataFileName); - re = re && Objects.equals(codecVersion, that.codecVersion); - re = re && Objects.equals(routingTableVersion, that.routingTableVersion); - re = re && Objects.equals(indicesRouting, that.indicesRouting); - re = re && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata); - re = re && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata); - re = re && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata); - re = re && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap); - re = re && Objects.equals(metadataVersion, that.metadataVersion); - re = re && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata); - re = re && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata); - re = re && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata); - re = re && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings); - re = re && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap); - re = re && Objects.equals(diffManifest, that.diffManifest); - return re; + return Objects.equals(indices, that.indices) + && clusterTerm == that.clusterTerm + && stateVersion == that.stateVersion + && Objects.equals(clusterUUID, that.clusterUUID) + && Objects.equals(stateUUID, that.stateUUID) + && Objects.equals(opensearchVersion, that.opensearchVersion) + && Objects.equals(nodeId, that.nodeId) + && Objects.equals(committed, that.committed) + && Objects.equals(previousClusterUUID, that.previousClusterUUID) + && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) + && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) + && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(routingTableVersion, that.routingTableVersion) + && Objects.equals(indicesRouting, that.indicesRouting) + && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) + && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) + && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) + && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) + && Objects.equals(metadataVersion, that.metadataVersion) + && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) + && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) + && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) + && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) + && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap) + && Objects.equals(diffManifest, that.diffManifest); } @Override diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java new file mode 100644 index 0000000000000..7c4ffa395fd49 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java @@ -0,0 +1,259 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.opensearch.Version.CURRENT; +import static org.opensearch.cluster.SnapshotsInProgress.State.INIT; +import static org.opensearch.cluster.SnapshotsInProgress.State.STARTED; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.RepositoryCleanupInProgress; +import org.opensearch.cluster.RepositoryCleanupInProgress.Entry; +import org.opensearch.cluster.SnapshotDeletionsInProgress; +import org.opensearch.cluster.SnapshotsInProgress; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.snapshots.Snapshot; +import org.opensearch.snapshots.SnapshotId; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; + +public class RemoteClusterStateCustomsTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final String CUSTOM_TYPE = "test-custom"; + private static final long STATE_VERSION = 3L; + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedWriteableRegistry namedWriteableRegistry; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedWriteableRegistry = writableRegistry(); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testClusterUUID() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + TEST_BLOB_NAME, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + TEST_BLOB_NAME, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + TEST_BLOB_NAME, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/clusterStateCustoms"; + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + uploadedFile, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[] { "user", "local", "opensearch", "clusterStateCustoms" })); + } + + public void testBlobPathParameters() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN))); + String expectedPrefix = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, "test-custom"); + assertThat(params.getFilePrefix(), is(expectedPrefix)); + } + + public void testGenerateBlobFileName() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + String expectedPrefix = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, "test-custom"); + assertThat(nameTokens[0], is(expectedPrefix)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(STATE_VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION))); + + } + + public void testGetUploadedMetadata() throws IOException { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + String expectedPrefix = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, "test-custom"); + assertThat(uploadedMetadata.getComponent(), is(expectedPrefix)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + SnapshotsInProgress.TYPE, + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertThat(inputStream.available(), greaterThan(0)); + Custom readclusterStateCustoms = remoteObjectForUpload.deserialize(inputStream); + assertThat(readclusterStateCustoms, is(clusterStateCustoms)); + } + } + + private Custom getClusterStateCustom() { + return SnapshotsInProgress.of(List.of(new SnapshotsInProgress.Entry( + new Snapshot("repo", new SnapshotId("test-snapshot", "test-snapshot-uuid")), + false, + false, + INIT, + emptyList(), + emptyList(), + 0L, + 0L, + emptyMap(), + emptyMap(), + CURRENT, + false + ))); + } +} From 955871a3d47d4e49a9945aa2c2d804c3b8e1482a Mon Sep 17 00:00:00 2001 From: Shivansh Arora Date: Mon, 10 Jun 2024 19:35:21 +0530 Subject: [PATCH 12/12] change codec version to V2 in tests Signed-off-by: Shivansh Arora --- .../remote/ClusterMetadataManifest.java | 48 ++++++------ .../RemoteClusterStateServiceTests.java | 17 +++-- .../model/RemoteClusterStateCustomsTests.java | 75 ++++++++++--------- 3 files changed, 71 insertions(+), 69 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 44c08ce6a09dc..9fe48b29c93a2 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -746,30 +746,30 @@ public boolean equals(Object o) { } final ClusterMetadataManifest that = (ClusterMetadataManifest) o; return Objects.equals(indices, that.indices) - && clusterTerm == that.clusterTerm - && stateVersion == that.stateVersion - && Objects.equals(clusterUUID, that.clusterUUID) - && Objects.equals(stateUUID, that.stateUUID) - && Objects.equals(opensearchVersion, that.opensearchVersion) - && Objects.equals(nodeId, that.nodeId) - && Objects.equals(committed, that.committed) - && Objects.equals(previousClusterUUID, that.previousClusterUUID) - && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) - && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) - && Objects.equals(codecVersion, that.codecVersion) - && Objects.equals(routingTableVersion, that.routingTableVersion) - && Objects.equals(indicesRouting, that.indicesRouting) - && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) - && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) - && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) - && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) - && Objects.equals(metadataVersion, that.metadataVersion) - && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) - && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) - && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) - && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) - && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap) - && Objects.equals(diffManifest, that.diffManifest); + && clusterTerm == that.clusterTerm + && stateVersion == that.stateVersion + && Objects.equals(clusterUUID, that.clusterUUID) + && Objects.equals(stateUUID, that.stateUUID) + && Objects.equals(opensearchVersion, that.opensearchVersion) + && Objects.equals(nodeId, that.nodeId) + && Objects.equals(committed, that.committed) + && Objects.equals(previousClusterUUID, that.previousClusterUUID) + && Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted) + && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) + && Objects.equals(codecVersion, that.codecVersion) + && Objects.equals(routingTableVersion, that.routingTableVersion) + && Objects.equals(indicesRouting, that.indicesRouting) + && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) + && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) + && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) + && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) + && Objects.equals(metadataVersion, that.metadataVersion) + && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) + && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) + && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) + && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) + && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap) + && Objects.equals(diffManifest, that.diffManifest); } @Override diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index 1c06b50da910d..044f3cc27d686 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -86,6 +86,7 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; @@ -475,7 +476,7 @@ public void testMigrationFromCodecV0ManifestToCodecV2Manifest() throws IOExcepti * even if it was not changed in this cluster state update */ public void testMigrationFromCodecV1ManifestToCodecV2Manifest() throws IOException { - verifyCodecMigrationManifest(ClusterMetadataManifest.CODEC_V1); + verifyCodecMigrationManifest(CODEC_V1); } private void verifyCodecMigrationManifest(int previousCodec) throws IOException { @@ -524,7 +525,7 @@ public void testWriteIncrementalGlobalMetadataFromCodecV0Success() throws IOExce public void testWriteIncrementalGlobalMetadataFromCodecV1Success() throws IOException { final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() - .codecVersion(1) + .codecVersion(CODEC_V1) .globalMetadataFileName("global-metadata-file") .indices(Collections.emptyList()) .build(); @@ -548,7 +549,7 @@ private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMe ); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() - .codecVersion(3) + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) .indices(Collections.emptyList()) .clusterTerm(1L) .stateVersion(1L) @@ -1105,7 +1106,7 @@ public void testReadGlobalMetadataIOException() throws IOException { .stateVersion(1L) .stateUUID("state-uuid") .clusterUUID("cluster-uuid") - .codecVersion(ClusterMetadataManifest.CODEC_V1) + .codecVersion(CODEC_V1) .globalMetadataFileName(globalIndexMetadataName) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) @@ -1314,7 +1315,7 @@ public void testFileNames() { assertThat(splittedIndexMetadataFileName[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION))); verifyManifestFileNameWithCodec(MANIFEST_CURRENT_CODEC_VERSION); - verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V1); + verifyManifestFileNameWithCodec(CODEC_V1); verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V0); } @@ -1577,7 +1578,7 @@ private ClusterMetadataManifest generateV1ClusterMetadataManifest( .committed(true) .clusterUUIDCommitted(isUUIDCommitted) .globalMetadataFileName(globalMetadataFileName) - .codecVersion(ClusterMetadataManifest.CODEC_V1) + .codecVersion(CODEC_V1) .build(); } @@ -1649,7 +1650,7 @@ private void mockBlobContainer( Map indexMetadataMap, int codecVersion ) throws IOException { - String manifestFileName = codecVersion >= ClusterMetadataManifest.CODEC_V1 + String manifestFileName = codecVersion >= CODEC_V1 ? "manifest__manifestFileName__abcd__abcd__abcd__" + codecVersion : "manifestFileName"; BlobMetadata blobMetadata = new PlainBlobMetadata(manifestFileName, 1); @@ -1769,7 +1770,7 @@ private void mockBlobContainerForGlobalMetadata( } ); } - } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + } else if (codecVersion == CODEC_V1) { String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))) .thenAnswer((invocationOnMock) -> { diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java index 7c4ffa395fd49..1f7a5e8bfffb1 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java @@ -8,30 +8,7 @@ package org.opensearch.gateway.remote.model; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.opensearch.Version.CURRENT; -import static org.opensearch.cluster.SnapshotsInProgress.State.INIT; -import static org.opensearch.cluster.SnapshotsInProgress.State.STARTED; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; -import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import org.junit.After; -import org.junit.Before; import org.opensearch.cluster.ClusterState.Custom; -import org.opensearch.cluster.RepositoryCleanupInProgress; -import org.opensearch.cluster.RepositoryCleanupInProgress.Entry; -import org.opensearch.cluster.SnapshotDeletionsInProgress; import org.opensearch.cluster.SnapshotsInProgress; import org.opensearch.common.blobstore.BlobPath; import org.opensearch.common.compress.DeflateCompressor; @@ -51,6 +28,26 @@ import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.opensearch.Version.CURRENT; +import static org.opensearch.cluster.SnapshotsInProgress.State.INIT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class RemoteClusterStateCustomsTests extends OpenSearchTestCase { private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; @@ -241,19 +238,23 @@ public void testSerDe() throws IOException { } private Custom getClusterStateCustom() { - return SnapshotsInProgress.of(List.of(new SnapshotsInProgress.Entry( - new Snapshot("repo", new SnapshotId("test-snapshot", "test-snapshot-uuid")), - false, - false, - INIT, - emptyList(), - emptyList(), - 0L, - 0L, - emptyMap(), - emptyMap(), - CURRENT, - false - ))); + return SnapshotsInProgress.of( + List.of( + new SnapshotsInProgress.Entry( + new Snapshot("repo", new SnapshotId("test-snapshot", "test-snapshot-uuid")), + false, + false, + INIT, + emptyList(), + emptyList(), + 0L, + 0L, + emptyMap(), + emptyMap(), + CURRENT, + false + ) + ) + ); } }