diff --git a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java index a38fc81bebc08..d21cd354bf659 100644 --- a/server/src/main/java/org/opensearch/cluster/DiffableUtils.java +++ b/server/src/main/java/org/opensearch/cluster/DiffableUtils.java @@ -494,6 +494,18 @@ public void writeDiff(Diff value, StreamOutput out) throws IOException { * @opensearch.internal */ public abstract static class NonDiffableValueSerializer implements ValueSerializer { + private static final NonDiffableValueSerializer ABSTRACT_INSTANCE = new NonDiffableValueSerializer<>() { + @Override + public void write(Object value, StreamOutput out) { + throw new UnsupportedOperationException(); + } + + @Override + public Object read(StreamInput in, Object key) { + throw new UnsupportedOperationException(); + } + }; + @Override public boolean supportsDiffableValues() { return false; @@ -513,6 +525,10 @@ public void writeDiff(Diff value, StreamOutput out) throws IOException { public Diff readDiff(StreamInput in, K key) throws IOException { throw new UnsupportedOperationException(); } + + public static NonDiffableValueSerializer getAbstractInstance() { + return ABSTRACT_INSTANCE; + } } /** diff --git a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java index a8102182576ff..5865891c8a7f9 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/DiffableStringMap.java @@ -66,7 +66,7 @@ public static DiffableStringMap readFrom(StreamInput in) throws IOException { return map.isEmpty() ? EMPTY : new DiffableStringMap(map); } - DiffableStringMap(final Map map) { + public DiffableStringMap(final Map map) { this.innerMap = Collections.unmodifiableMap(map); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java index e1aa5626f36c1..232f900f25375 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/Metadata.java @@ -973,6 +973,10 @@ public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metad return metadata1.persistentSettings.equals(metadata2.persistentSettings); } + public static boolean isTransientSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) { + return metadata1.transientSettings.equals(metadata2.transientSettings); + } + public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) { return metadata1.templates.equals(metadata2.templates); } diff --git a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java index dbf01904116ed..c8ba91fab6871 100644 --- a/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java +++ b/server/src/main/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableService.java @@ -28,7 +28,7 @@ * Interface for RemoteRoutingTableService. Exposes methods to orchestrate upload and download of routing table from remote store. */ public interface RemoteRoutingTableService extends LifecycleComponent { - static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = + public static final DiffableUtils.NonDiffableValueSerializer CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER = new DiffableUtils.NonDiffableValueSerializer() { @Override public void write(IndexRoutingTable value, StreamOutput out) throws IOException { diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java index 4f18f1591a3fd..1fca6d959fbbf 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java @@ -40,8 +40,9 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { public static final int CODEC_V0 = 0; // Older codec version, where we haven't introduced codec versions for manifest. public static final int CODEC_V1 = 1; // In Codec V1 we have introduced global-metadata and codec version in Manifest file. - public static final int CODEC_V2 = 2; // In Codec V2, there are seperate metadata files rather than a single global metadata file. - public static final int CODEC_V3 = 3; // In Codec V3, we introduce index routing-metadata in manifest file. + public static final int CODEC_V2 = 2; // In Codec V2, there are separate metadata files rather than a single global metadata file, + // also we introduce index routing-metadata, diff and other attributes as part of manifest + // required for state publication private static final ParseField CLUSTER_TERM_FIELD = new ParseField("cluster_term"); private static final ParseField STATE_VERSION_FIELD = new ParseField("state_version"); @@ -61,6 +62,15 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment { private static final ParseField UPLOADED_CUSTOM_METADATA = new ParseField("uploaded_custom_metadata"); private static final ParseField ROUTING_TABLE_VERSION_FIELD = new ParseField("routing_table_version"); private static final ParseField INDICES_ROUTING_FIELD = new ParseField("indices_routing"); + private static final ParseField METADATA_VERSION = new ParseField("metadata_version"); + private static final ParseField UPLOADED_TRANSIENT_SETTINGS_METADATA = new ParseField("uploaded_transient_settings_metadata"); + private static final ParseField UPLOADED_DISCOVERY_NODES_METADATA = new ParseField("uploaded_discovery_nodes_metadata"); + private static final ParseField UPLOADED_CLUSTER_BLOCKS_METADATA = new ParseField("uploaded_cluster_blocks_metadata"); + private static final ParseField UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA = new ParseField( + "uploaded_hashes_of_consistent_settings_metadata" + ); + private static final ParseField UPLOADED_CLUSTER_STATE_CUSTOM_METADATA = new ParseField("uploaded_cluster_state_custom_metadata"); + private static final ParseField DIFF_MANIFEST = new ParseField("diff_manifest"); private static ClusterMetadataManifest.Builder manifestV0Builder(Object[] fields) { return ClusterMetadataManifest.builder() @@ -86,13 +96,16 @@ private static ClusterMetadataManifest.Builder manifestV2Builder(Object[] fields .coordinationMetadata(coordinationMetadata(fields)) .settingMetadata(settingsMetadata(fields)) .templatesMetadata(templatesMetadata(fields)) - .customMetadataMap(customMetadata(fields)); - } - - private static ClusterMetadataManifest.Builder manifestV3Builder(Object[] fields) { - return manifestV2Builder(fields).codecVersion(codecVersion(fields)) + .customMetadataMap(customMetadata(fields)) .routingTableVersion(routingTableVersion(fields)) - .indicesRouting(indicesRouting(fields)); + .indicesRouting(indicesRouting(fields)) + .discoveryNodesMetadata(discoveryNodesMetadata(fields)) + .clusterBlocksMetadata(clusterBlocksMetadata(fields)) + .diffManifest(diffManifest(fields)) + .metadataVersion(metadataVersion(fields)) + .transientSettingsMetadata(transientSettingsMetadata(fields)) + .hashesOfConsistentSettings(hashesOfConsistentSettings(fields)) + .clusterStateCustomMetadataMap(clusterStateCustomMetadata(fields)); } private static long term(Object[] fields) { @@ -168,6 +181,35 @@ private static List indicesRouting(Object[] fields) { return (List) fields[16]; } + private static UploadedMetadataAttribute discoveryNodesMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[17]; + } + + private static UploadedMetadataAttribute clusterBlocksMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[18]; + } + + private static long metadataVersion(Object[] fields) { + return (long) fields[19]; + } + + private static UploadedMetadataAttribute transientSettingsMetadata(Object[] fields) { + return (UploadedMetadataAttribute) fields[20]; + } + + private static UploadedMetadataAttribute hashesOfConsistentSettings(Object[] fields) { + return (UploadedMetadataAttribute) fields[21]; + } + + private static Map clusterStateCustomMetadata(Object[] fields) { + List customs = (List) fields[22]; + return customs.stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())); + } + + private static ClusterStateDiffManifest diffManifest(Object[] fields) { + return (ClusterStateDiffManifest) fields[23]; + } + private static final ConstructingObjectParser PARSER_V0 = new ConstructingObjectParser<>( "cluster_metadata_manifest", fields -> manifestV0Builder(fields).build() @@ -183,18 +225,12 @@ private static List indicesRouting(Object[] fields) { fields -> manifestV2Builder(fields).build() ); - private static final ConstructingObjectParser PARSER_V3 = new ConstructingObjectParser<>( - "cluster_metadata_manifest", - fields -> manifestV3Builder(fields).build() - ); - - private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V3; + private static final ConstructingObjectParser CURRENT_PARSER = PARSER_V2; static { declareParser(PARSER_V0, CODEC_V0); declareParser(PARSER_V1, CODEC_V1); declareParser(PARSER_V2, CODEC_V2); - declareParser(PARSER_V3, CODEC_V3); } private static void declareParser(ConstructingObjectParser parser, long codec_version) { @@ -238,14 +274,43 @@ private static void declareParser(ConstructingObjectParser= CODEC_V3) { parser.declareLong(ConstructingObjectParser.constructorArg(), ROUTING_TABLE_VERSION_FIELD); parser.declareObjectArray( ConstructingObjectParser.constructorArg(), (p, c) -> UploadedIndexMetadata.fromXContent(p), INDICES_ROUTING_FIELD ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_DISCOVERY_NODES_METADATA + ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_CLUSTER_BLOCKS_METADATA + ); + parser.declareLong(ConstructingObjectParser.constructorArg(), METADATA_VERSION); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_TRANSIENT_SETTINGS_METADATA + ); + parser.declareNamedObject( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA + ); + parser.declareNamedObjects( + ConstructingObjectParser.optionalConstructorArg(), + UploadedMetadataAttribute.PARSER, + UPLOADED_CLUSTER_STATE_CUSTOM_METADATA + ); + parser.declareObject( + ConstructingObjectParser.optionalConstructorArg(), + (p, c) -> ClusterStateDiffManifest.fromXContent(p), + DIFF_MANIFEST + ); } } @@ -267,6 +332,13 @@ private static void declareParser(ConstructingObjectParser indicesRouting; + private final long metadataVersion; + private final UploadedMetadataAttribute uploadedTransientSettingsMetadata; + private final UploadedMetadataAttribute uploadedDiscoveryNodesMetadata; + private final UploadedMetadataAttribute uploadedClusterBlocksMetadata; + private final UploadedMetadataAttribute uploadedHashesOfConsistentSettings; + private final Map uploadedClusterStateCustomMap; + private final ClusterStateDiffManifest diffManifest; public List getIndices() { return indices; @@ -332,9 +404,32 @@ public Map getCustomMetadataMap() { return uploadedCustomMetadataMap; } - // TODO https://github.com/opensearch-project/OpenSearch/pull/14089 + public long getMetadataVersion() { + return metadataVersion; + } + + public UploadedMetadataAttribute getTransientSettingsMetadata() { + return uploadedTransientSettingsMetadata; + } + + public UploadedMetadataAttribute getDiscoveryNodesMetadata() { + return uploadedDiscoveryNodesMetadata; + } + + public UploadedMetadataAttribute getClusterBlocksMetadata() { + return uploadedClusterBlocksMetadata; + } + public ClusterStateDiffManifest getDiffManifest() { - return new ClusterStateDiffManifest(); + return diffManifest; + } + + public Map getClusterStateCustomMap() { + return uploadedClusterStateCustomMap; + } + + public UploadedMetadataAttribute getHashesOfConsistentSettings() { + return uploadedHashesOfConsistentSettings; } public boolean hasMetadataAttributesFiles() { @@ -370,7 +465,14 @@ public ClusterMetadataManifest( UploadedMetadataAttribute uploadedTemplatesMetadata, Map uploadedCustomMetadataMap, long routingTableVersion, - List indicesRouting + List indicesRouting, + long metadataVersion, + UploadedMetadataAttribute discoveryNodesMetadata, + UploadedMetadataAttribute clusterBlocksMetadata, + UploadedMetadataAttribute uploadedTransientSettingsMetadata, + UploadedMetadataAttribute uploadedHashesOfConsistentSettings, + Map uploadedClusterStateCustomMap, + ClusterStateDiffManifest diffManifest ) { this.clusterTerm = clusterTerm; this.stateVersion = version; @@ -392,6 +494,15 @@ public ClusterMetadataManifest( this.uploadedCustomMetadataMap = Collections.unmodifiableMap( uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>() ); + this.uploadedDiscoveryNodesMetadata = discoveryNodesMetadata; + this.uploadedClusterBlocksMetadata = clusterBlocksMetadata; + this.diffManifest = diffManifest; + this.metadataVersion = metadataVersion; + this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata; + this.uploadedHashesOfConsistentSettings = uploadedHashesOfConsistentSettings; + this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( + uploadedClusterStateCustomMap != null ? uploadedClusterStateCustomMap : new HashMap<>() + ); } public ClusterMetadataManifest(StreamInput in) throws IOException { @@ -416,24 +527,56 @@ public ClusterMetadataManifest(StreamInput in) throws IOException { this.globalMetadataFileName = null; this.routingTableVersion = in.readLong(); this.indicesRouting = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new)); - } else if (in.getVersion().onOrAfter(Version.V_2_12_0)) { - this.codecVersion = in.readInt(); - this.globalMetadataFileName = in.readString(); - this.uploadedCoordinationMetadata = null; - this.uploadedSettingsMetadata = null; - this.uploadedTemplatesMetadata = null; - this.uploadedCustomMetadataMap = null; - this.routingTableVersion = -1; - this.indicesRouting = null; + this.metadataVersion = in.readLong(); + if (in.readBoolean()) { + this.uploadedDiscoveryNodesMetadata = new UploadedMetadataAttribute(in); + } else { + this.uploadedDiscoveryNodesMetadata = null; + } + if (in.readBoolean()) { + this.uploadedClusterBlocksMetadata = new UploadedMetadataAttribute(in); + } else { + this.uploadedClusterBlocksMetadata = null; + } + if (in.readBoolean()) { + this.uploadedTransientSettingsMetadata = new UploadedMetadataAttribute(in); + } else { + this.uploadedTransientSettingsMetadata = null; + } + if (in.readBoolean()) { + this.uploadedHashesOfConsistentSettings = new UploadedMetadataAttribute(in); + } else { + this.uploadedHashesOfConsistentSettings = null; + } + this.uploadedClusterStateCustomMap = Collections.unmodifiableMap( + in.readMap(StreamInput::readString, UploadedMetadataAttribute::new) + ); + if (in.readBoolean()) { + this.diffManifest = new ClusterStateDiffManifest(in); + } else { + this.diffManifest = null; + } } else { - this.codecVersion = CODEC_V0; // Default codec - this.globalMetadataFileName = null; + if (in.getVersion().onOrAfter(Version.V_2_12_0)) { + this.codecVersion = in.readInt(); + this.globalMetadataFileName = in.readString(); + } else { + this.codecVersion = CODEC_V0; // Default codec + this.globalMetadataFileName = null; + } this.uploadedCoordinationMetadata = null; this.uploadedSettingsMetadata = null; this.uploadedTemplatesMetadata = null; this.uploadedCustomMetadataMap = null; this.routingTableVersion = -1; this.indicesRouting = null; + this.uploadedDiscoveryNodesMetadata = null; + this.uploadedClusterBlocksMetadata = null; + this.diffManifest = null; + this.metadataVersion = -1; + this.uploadedTransientSettingsMetadata = null; + this.uploadedHashesOfConsistentSettings = null; + this.uploadedClusterStateCustomMap = null; } } @@ -487,11 +630,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws attribute.toXContent(builder, params); } builder.endObject(); - } else if (onOrAfterCodecVersion(CODEC_V1)) { - builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); - builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); - } - if (onOrAfterCodecVersion(CODEC_V3)) { builder.field(ROUTING_TABLE_VERSION_FIELD.getPreferredName(), getRoutingTableVersion()); builder.startArray(INDICES_ROUTING_FIELD.getPreferredName()); { @@ -502,6 +640,40 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endArray(); + if (getDiscoveryNodesMetadata() != null) { + builder.startObject(UPLOADED_DISCOVERY_NODES_METADATA.getPreferredName()); + getDiscoveryNodesMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getClusterBlocksMetadata() != null) { + builder.startObject(UPLOADED_CLUSTER_BLOCKS_METADATA.getPreferredName()); + getClusterBlocksMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getTransientSettingsMetadata() != null) { + builder.startObject(UPLOADED_TRANSIENT_SETTINGS_METADATA.getPreferredName()); + getTransientSettingsMetadata().toXContent(builder, params); + builder.endObject(); + } + if (getDiffManifest() != null) { + builder.startObject(DIFF_MANIFEST.getPreferredName()); + getDiffManifest().toXContent(builder, params); + builder.endObject(); + } + builder.field(METADATA_VERSION.getPreferredName(), getMetadataVersion()); + if (getHashesOfConsistentSettings() != null) { + builder.startObject(UPLOADED_HASHES_OF_CONSISTENT_SETTINGS_METADATA.getPreferredName()); + getHashesOfConsistentSettings().toXContent(builder, params); + builder.endObject(); + } + builder.startObject(UPLOADED_CLUSTER_STATE_CUSTOM_METADATA.getPreferredName()); + for (UploadedMetadataAttribute attribute : getClusterStateCustomMap().values()) { + attribute.toXContent(builder, params); + } + builder.endObject(); + } else if (onOrAfterCodecVersion(CODEC_V1)) { + builder.field(CODEC_VERSION_FIELD.getPreferredName(), getCodecVersion()); + builder.field(GLOBAL_METADATA_FIELD.getPreferredName(), getGlobalMetadataFileName()); } return builder; } @@ -526,6 +698,38 @@ public void writeTo(StreamOutput out) throws IOException { out.writeMap(uploadedCustomMetadataMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); out.writeLong(routingTableVersion); out.writeCollection(indicesRouting); + out.writeLong(metadataVersion); + if (uploadedDiscoveryNodesMetadata != null) { + out.writeBoolean(true); + uploadedDiscoveryNodesMetadata.writeTo(out); + } else { + out.writeBoolean(false); + } + if (uploadedClusterBlocksMetadata != null) { + out.writeBoolean(true); + uploadedClusterBlocksMetadata.writeTo(out); + } else { + out.writeBoolean(false); + } + if (uploadedTransientSettingsMetadata != null) { + out.writeBoolean(true); + uploadedTransientSettingsMetadata.writeTo(out); + } else { + out.writeBoolean(false); + } + if (uploadedHashesOfConsistentSettings != null) { + out.writeBoolean(true); + uploadedHashesOfConsistentSettings.writeTo(out); + } else { + out.writeBoolean(false); + } + out.writeMap(uploadedClusterStateCustomMap, StreamOutput::writeString, (o, v) -> v.writeTo(o)); + if (diffManifest != null) { + out.writeBoolean(true); + diffManifest.writeTo(out); + } else { + out.writeBoolean(false); + } } else if (out.getVersion().onOrAfter(Version.V_2_12_0)) { out.writeInt(codecVersion); out.writeString(globalMetadataFileName); @@ -554,7 +758,18 @@ public boolean equals(Object o) { && Objects.equals(globalMetadataFileName, that.globalMetadataFileName) && Objects.equals(codecVersion, that.codecVersion) && Objects.equals(routingTableVersion, that.routingTableVersion) - && Objects.equals(indicesRouting, that.indicesRouting); + && Objects.equals(indicesRouting, that.indicesRouting) + && Objects.equals(uploadedCoordinationMetadata, that.uploadedCoordinationMetadata) + && Objects.equals(uploadedSettingsMetadata, that.uploadedSettingsMetadata) + && Objects.equals(uploadedTemplatesMetadata, that.uploadedTemplatesMetadata) + && Objects.equals(uploadedCustomMetadataMap, that.uploadedCustomMetadataMap) + && Objects.equals(metadataVersion, that.metadataVersion) + && Objects.equals(uploadedDiscoveryNodesMetadata, that.uploadedDiscoveryNodesMetadata) + && Objects.equals(uploadedClusterBlocksMetadata, that.uploadedClusterBlocksMetadata) + && Objects.equals(uploadedTransientSettingsMetadata, that.uploadedTransientSettingsMetadata) + && Objects.equals(uploadedHashesOfConsistentSettings, that.uploadedHashesOfConsistentSettings) + && Objects.equals(uploadedClusterStateCustomMap, that.uploadedClusterStateCustomMap) + && Objects.equals(diffManifest, that.diffManifest); } @Override @@ -573,7 +788,18 @@ public int hashCode() { previousClusterUUID, clusterUUIDCommitted, routingTableVersion, - indicesRouting + indicesRouting, + uploadedCoordinationMetadata, + uploadedSettingsMetadata, + uploadedTemplatesMetadata, + uploadedCustomMetadataMap, + metadataVersion, + uploadedDiscoveryNodesMetadata, + uploadedClusterBlocksMetadata, + uploadedTransientSettingsMetadata, + uploadedHashesOfConsistentSettings, + uploadedClusterStateCustomMap, + diffManifest ); } @@ -627,6 +853,13 @@ public static class Builder { private boolean clusterUUIDCommitted; private long routingTableVersion; private List indicesRouting; + private long metadataVersion; + private UploadedMetadataAttribute discoveryNodesMetadata; + private UploadedMetadataAttribute clusterBlocksMetadata; + private UploadedMetadataAttribute transientSettingsMetadata; + private UploadedMetadataAttribute hashesOfConsistentSettings; + private Map clusterStateCustomMetadataMap; + private ClusterStateDiffManifest diffManifest; public Builder indices(List indices) { this.indices = indices; @@ -731,10 +964,46 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) { return this; } + public Builder metadataVersion(long metadataVersion) { + this.metadataVersion = metadataVersion; + return this; + } + + public Builder discoveryNodesMetadata(UploadedMetadataAttribute discoveryNodesMetadata) { + this.discoveryNodesMetadata = discoveryNodesMetadata; + return this; + } + + public Builder clusterBlocksMetadata(UploadedMetadataAttribute clusterBlocksMetadata) { + this.clusterBlocksMetadata = clusterBlocksMetadata; + return this; + } + + public Builder transientSettingsMetadata(UploadedMetadataAttribute settingsMetadata) { + this.transientSettingsMetadata = settingsMetadata; + return this; + } + + public Builder hashesOfConsistentSettings(UploadedMetadataAttribute hashesOfConsistentSettings) { + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + return this; + } + + public Builder clusterStateCustomMetadataMap(Map clusterStateCustomMetadataMap) { + this.clusterStateCustomMetadataMap = clusterStateCustomMetadataMap; + return this; + } + + public Builder diffManifest(ClusterStateDiffManifest diffManifest) { + this.diffManifest = diffManifest; + return this; + } + public Builder() { indices = new ArrayList<>(); customMetadataMap = new HashMap<>(); indicesRouting = new ArrayList<>(); + clusterStateCustomMetadataMap = new HashMap<>(); } public Builder(ClusterMetadataManifest manifest) { @@ -756,6 +1025,12 @@ public Builder(ClusterMetadataManifest manifest) { this.clusterUUIDCommitted = manifest.clusterUUIDCommitted; this.routingTableVersion = manifest.routingTableVersion; this.indicesRouting = new ArrayList<>(manifest.indicesRouting); + this.discoveryNodesMetadata = manifest.uploadedDiscoveryNodesMetadata; + this.clusterBlocksMetadata = manifest.uploadedClusterBlocksMetadata; + this.transientSettingsMetadata = manifest.uploadedTransientSettingsMetadata; + this.diffManifest = manifest.diffManifest; + this.hashesOfConsistentSettings = manifest.uploadedHashesOfConsistentSettings; + this.clusterStateCustomMetadataMap = manifest.uploadedClusterStateCustomMap; } public ClusterMetadataManifest build() { @@ -777,7 +1052,14 @@ public ClusterMetadataManifest build() { templatesMetadata, customMetadataMap, routingTableVersion, - indicesRouting + indicesRouting, + metadataVersion, + discoveryNodesMetadata, + clusterBlocksMetadata, + transientSettingsMetadata, + hashesOfConsistentSettings, + clusterStateCustomMetadataMap, + diffManifest ); } @@ -1004,6 +1286,19 @@ public static UploadedMetadataAttribute fromXContent(XContentParser parser) thro return PARSER.parse(parser, null, parser.currentName()); } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UploadedMetadataAttribute that = (UploadedMetadataAttribute) o; + return Objects.equals(attributeName, that.attributeName) && Objects.equals(uploadedFilename, that.uploadedFilename); + } + + @Override + public int hashCode() { + return Objects.hash(attributeName, uploadedFilename); + } + @Override public String toString() { return "UploadedMetadataAttribute{" diff --git a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java index d59c6042e7834..65ae2675a95da 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/ClusterStateDiffManifest.java @@ -8,15 +8,668 @@ package org.opensearch.gateway.remote; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.DiffableUtils; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.routing.IndexRoutingTable; +import org.opensearch.core.common.Strings; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.MediaTypeRegistry; +import org.opensearch.core.xcontent.ToXContentFragment; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParseException; +import org.opensearch.core.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.opensearch.cluster.DiffableUtils.NonDiffableValueSerializer.getAbstractInstance; +import static org.opensearch.cluster.DiffableUtils.getStringKeySerializer; +import static org.opensearch.cluster.routing.remote.RemoteRoutingTableService.CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER; +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + /** * Manifest of diff between two cluster states * * @opensearch.internal */ -public class ClusterStateDiffManifest { +public class ClusterStateDiffManifest implements ToXContentFragment, Writeable { + private static final String FROM_STATE_UUID_FIELD = "from_state_uuid"; + private static final String TO_STATE_UUID_FIELD = "to_state_uuid"; + private static final String METADATA_DIFF_FIELD = "metadata_diff"; + private static final String COORDINATION_METADATA_UPDATED_FIELD = "coordination_metadata_diff"; + private static final String SETTINGS_METADATA_UPDATED_FIELD = "settings_metadata_diff"; + private static final String TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD = "transient_settings_metadata_diff"; + private static final String TEMPLATES_METADATA_UPDATED_FIELD = "templates_metadata_diff"; + private static final String HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD = "hashes_of_consistent_settings_diff"; + private static final String INDICES_DIFF_FIELD = "indices_diff"; + private static final String METADATA_CUSTOM_DIFF_FIELD = "metadata_custom_diff"; + private static final String UPSERTS_FIELD = "upserts"; + private static final String DELETES_FIELD = "deletes"; + private static final String CLUSTER_BLOCKS_UPDATED_FIELD = "cluster_blocks_diff"; + private static final String DISCOVERY_NODES_UPDATED_FIELD = "discovery_nodes_diff"; + private static final String ROUTING_TABLE_DIFF = "routing_table_diff"; + private static final String CLUSTER_STATE_CUSTOM_DIFF_FIELD = "cluster_state_custom_diff"; + + private final String fromStateUUID; + private final String toStateUUID; + private final boolean coordinationMetadataUpdated; + private final boolean settingsMetadataUpdated; + private final boolean transientSettingsMetadataUpdated; + private final boolean templatesMetadataUpdated; + private final List indicesUpdated; + private final List indicesDeleted; + private final List customMetadataUpdated; + private final List customMetadataDeleted; + private final boolean clusterBlocksUpdated; + private final boolean discoveryNodesUpdated; + private final List indicesRoutingUpdated; + private final List indicesRoutingDeleted; + private final boolean hashesOfConsistentSettingsUpdated; + private final List clusterStateCustomUpdated; + private final List clusterStateCustomDeleted; + + public ClusterStateDiffManifest(ClusterState state, ClusterState previousState) { + fromStateUUID = previousState.stateUUID(); + toStateUUID = state.stateUUID(); + coordinationMetadataUpdated = !Metadata.isCoordinationMetadataEqual(state.metadata(), previousState.metadata()); + settingsMetadataUpdated = !Metadata.isSettingsMetadataEqual(state.metadata(), previousState.metadata()); + transientSettingsMetadataUpdated = !Metadata.isTransientSettingsMetadataEqual(state.metadata(), previousState.metadata()); + templatesMetadataUpdated = !Metadata.isTemplatesMetadataEqual(state.metadata(), previousState.metadata()); + DiffableUtils.MapDiff> indicesDiff = DiffableUtils.diff( + previousState.metadata().indices(), + state.metadata().indices(), + getStringKeySerializer() + ); + indicesDeleted = indicesDiff.getDeletes(); + indicesUpdated = new ArrayList<>(indicesDiff.getDiffs().keySet()); + indicesUpdated.addAll(indicesDiff.getUpserts().keySet()); + clusterBlocksUpdated = !state.blocks().equals(previousState.blocks()); + discoveryNodesUpdated = state.nodes().delta(previousState.nodes()).hasChanges(); + DiffableUtils.MapDiff> customDiff = DiffableUtils.diff( + previousState.metadata().customs(), + state.metadata().customs(), + getStringKeySerializer(), + getAbstractInstance() + ); + customMetadataUpdated = new ArrayList<>(customDiff.getDiffs().keySet()); + customMetadataUpdated.addAll(customDiff.getUpserts().keySet()); + customMetadataDeleted = customDiff.getDeletes(); + + DiffableUtils.MapDiff> routingTableDiff = DiffableUtils.diff( + previousState.getRoutingTable().getIndicesRouting(), + state.getRoutingTable().getIndicesRouting(), + DiffableUtils.getStringKeySerializer(), + CUSTOM_ROUTING_TABLE_VALUE_SERIALIZER + ); + + indicesRoutingUpdated = new ArrayList<>(); + routingTableDiff.getUpserts().forEach((k, v) -> indicesRoutingUpdated.add(k)); + + indicesRoutingDeleted = routingTableDiff.getDeletes(); + hashesOfConsistentSettingsUpdated = !state.metadata() + .hashesOfConsistentSettings() + .equals(previousState.metadata().hashesOfConsistentSettings()); + DiffableUtils.MapDiff> clusterStateCustomDiff = DiffableUtils.diff( + previousState.customs(), + state.customs(), + getStringKeySerializer(), + getAbstractInstance() + ); + clusterStateCustomUpdated = new ArrayList<>(clusterStateCustomDiff.getDiffs().keySet()); + clusterStateCustomUpdated.addAll(clusterStateCustomDiff.getUpserts().keySet()); + clusterStateCustomDeleted = clusterStateCustomDiff.getDeletes(); + } + + public ClusterStateDiffManifest( + String fromStateUUID, + String toStateUUID, + boolean coordinationMetadataUpdated, + boolean settingsMetadataUpdated, + boolean transientSettingsMetadataUpdate, + boolean templatesMetadataUpdated, + List customMetadataUpdated, + List customMetadataDeleted, + List indicesUpdated, + List indicesDeleted, + boolean clusterBlocksUpdated, + boolean discoveryNodesUpdated, + List indicesRoutingUpdated, + List indicesRoutingDeleted, + boolean hashesOfConsistentSettingsUpdated, + List clusterStateCustomUpdated, + List clusterStateCustomDeleted + ) { + this.fromStateUUID = fromStateUUID; + this.toStateUUID = toStateUUID; + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + this.settingsMetadataUpdated = settingsMetadataUpdated; + this.transientSettingsMetadataUpdated = transientSettingsMetadataUpdate; + this.templatesMetadataUpdated = templatesMetadataUpdated; + this.customMetadataUpdated = customMetadataUpdated; + this.customMetadataDeleted = customMetadataDeleted; + this.indicesUpdated = indicesUpdated; + this.indicesDeleted = indicesDeleted; + this.clusterBlocksUpdated = clusterBlocksUpdated; + this.discoveryNodesUpdated = discoveryNodesUpdated; + this.indicesRoutingUpdated = indicesRoutingUpdated; + this.indicesRoutingDeleted = indicesRoutingDeleted; + this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated; + this.clusterStateCustomUpdated = clusterStateCustomUpdated; + this.clusterStateCustomDeleted = clusterStateCustomDeleted; + } + + public ClusterStateDiffManifest(StreamInput in) throws IOException { + this.fromStateUUID = in.readString(); + this.toStateUUID = in.readString(); + this.coordinationMetadataUpdated = in.readBoolean(); + this.settingsMetadataUpdated = in.readBoolean(); + this.transientSettingsMetadataUpdated = in.readBoolean(); + this.templatesMetadataUpdated = in.readBoolean(); + this.indicesUpdated = in.readStringList(); + this.indicesDeleted = in.readStringList(); + this.customMetadataUpdated = in.readStringList(); + this.customMetadataDeleted = in.readStringList(); + this.clusterBlocksUpdated = in.readBoolean(); + this.discoveryNodesUpdated = in.readBoolean(); + this.indicesRoutingUpdated = in.readStringList(); + this.indicesRoutingDeleted = in.readStringList(); + this.hashesOfConsistentSettingsUpdated = in.readBoolean(); + this.clusterStateCustomUpdated = in.readStringList(); + this.clusterStateCustomDeleted = in.readStringList(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(FROM_STATE_UUID_FIELD, fromStateUUID); + builder.field(TO_STATE_UUID_FIELD, toStateUUID); + builder.startObject(METADATA_DIFF_FIELD); + { + builder.field(COORDINATION_METADATA_UPDATED_FIELD, coordinationMetadataUpdated); + builder.field(SETTINGS_METADATA_UPDATED_FIELD, settingsMetadataUpdated); + builder.field(TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD, transientSettingsMetadataUpdated); + builder.field(TEMPLATES_METADATA_UPDATED_FIELD, templatesMetadataUpdated); + builder.startObject(INDICES_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String index : indicesUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String index : indicesDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + builder.startObject(METADATA_CUSTOM_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String custom : customMetadataUpdated) { + builder.value(custom); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String custom : customMetadataDeleted) { + builder.value(custom); + } + builder.endArray(); + builder.endObject(); + builder.field(HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD, hashesOfConsistentSettingsUpdated); + } + builder.endObject(); + builder.field(CLUSTER_BLOCKS_UPDATED_FIELD, clusterBlocksUpdated); + builder.field(DISCOVERY_NODES_UPDATED_FIELD, discoveryNodesUpdated); + + builder.startObject(ROUTING_TABLE_DIFF); + builder.startArray(UPSERTS_FIELD); + for (String index : indicesRoutingUpdated) { + builder.value(index); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String index : indicesRoutingDeleted) { + builder.value(index); + } + builder.endArray(); + builder.endObject(); + builder.startObject(CLUSTER_STATE_CUSTOM_DIFF_FIELD); + builder.startArray(UPSERTS_FIELD); + for (String custom : clusterStateCustomUpdated) { + builder.value(custom); + } + builder.endArray(); + builder.startArray(DELETES_FIELD); + for (String custom : clusterStateCustomDeleted) { + builder.value(custom); + } + builder.endArray(); + builder.endObject(); + return builder; + } + + public static ClusterStateDiffManifest fromXContent(XContentParser parser) throws IOException { + Builder builder = new Builder(); + if (parser.currentToken() == null) { // fresh parser? move to next token + parser.nextToken(); + } + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser); + String currentFieldName = parser.currentName(); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(METADATA_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (token.isValue()) { + switch (currentFieldName) { + case COORDINATION_METADATA_UPDATED_FIELD: + builder.coordinationMetadataUpdated(parser.booleanValue()); + break; + case SETTINGS_METADATA_UPDATED_FIELD: + builder.settingsMetadataUpdated(parser.booleanValue()); + break; + case TRANSIENT_SETTINGS_METADATA_UPDATED_FIELD: + builder.transientSettingsMetadataUpdate(parser.booleanValue()); + break; + case TEMPLATES_METADATA_UPDATED_FIELD: + builder.templatesMetadataUpdated(parser.booleanValue()); + break; + case HASHES_OF_CONSISTENT_SETTINGS_UPDATED_FIELD: + builder.hashesOfConsistentSettingsUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (currentFieldName.equals(INDICES_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.indicesUpdated(convertListToString(parser.listOrderedMap())); + break; + case DELETES_FIELD: + builder.indicesDeleted(convertListToString(parser.listOrderedMap())); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else if (currentFieldName.equals(METADATA_CUSTOM_DIFF_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + token = parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.customMetadataUpdated(convertListToString(parser.listOrderedMap())); + break; + case DELETES_FIELD: + builder.customMetadataDeleted(convertListToString(parser.listOrderedMap())); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + } else if (currentFieldName.equals(ROUTING_TABLE_DIFF)) { + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.indicesRoutingUpdated(convertListToString(parser.listOrderedMap())); + break; + case DELETES_FIELD: + builder.indicesRoutingDeleted(convertListToString(parser.listOrderedMap())); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else if (currentFieldName.equals(CLUSTER_STATE_CUSTOM_DIFF_FIELD)) { + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + currentFieldName = parser.currentName(); + parser.nextToken(); + switch (currentFieldName) { + case UPSERTS_FIELD: + builder.clusterStateCustomUpdated(convertListToString(parser.listOrderedMap())); + break; + case DELETES_FIELD: + builder.clusterStateCustomDeleted(convertListToString(parser.listOrderedMap())); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } + } else { + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else if (token.isValue()) { + switch (currentFieldName) { + case FROM_STATE_UUID_FIELD: + builder.fromStateUUID(parser.text()); + break; + case TO_STATE_UUID_FIELD: + builder.toStateUUID(parser.text()); + break; + case CLUSTER_BLOCKS_UPDATED_FIELD: + builder.clusterBlocksUpdated(parser.booleanValue()); + break; + case DISCOVERY_NODES_UPDATED_FIELD: + builder.discoveryNodesUpdated(parser.booleanValue()); + break; + default: + throw new XContentParseException("Unexpected field [" + currentFieldName + "]"); + } + } else { + throw new XContentParseException("Unexpected token [" + token + "]"); + } + } + return builder.build(); + } + + @Override + public String toString() { + return Strings.toString(MediaTypeRegistry.JSON, this); + } + + private static List convertListToString(List list) { + List convertedList = new ArrayList<>(); + for (Object o : list) { + convertedList.add(o.toString()); + } + return convertedList; + } - // TODO https://github.com/opensearch-project/OpenSearch/pull/14089 public String getFromStateUUID() { - return null; + return fromStateUUID; + } + + public String getToStateUUID() { + return toStateUUID; + } + + public boolean isCoordinationMetadataUpdated() { + return coordinationMetadataUpdated; + } + + public boolean isSettingsMetadataUpdated() { + return settingsMetadataUpdated; + } + + public boolean isTransientSettingsMetadataUpdated() { + return transientSettingsMetadataUpdated; + } + + public boolean isTemplatesMetadataUpdated() { + return templatesMetadataUpdated; + } + + public List getCustomMetadataUpdated() { + return customMetadataUpdated; + } + + public List getCustomMetadataDeleted() { + return customMetadataDeleted; + } + + public List getIndicesUpdated() { + return indicesUpdated; + } + + public List getIndicesDeleted() { + return indicesDeleted; + } + + public boolean isClusterBlocksUpdated() { + return clusterBlocksUpdated; + } + + public boolean isDiscoveryNodesUpdated() { + return discoveryNodesUpdated; + } + + public boolean isHashesOfConsistentSettingsUpdated() { + return hashesOfConsistentSettingsUpdated; + } + + public List getIndicesRoutingUpdated() { + return indicesRoutingUpdated; + } + + public List getIndicesRoutingDeleted() { + return indicesRoutingDeleted; + } + + public List getClusterStateCustomUpdated() { + return clusterStateCustomUpdated; + } + + public List getClusterStateCustomDeleted() { + return clusterStateCustomDeleted; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClusterStateDiffManifest that = (ClusterStateDiffManifest) o; + return coordinationMetadataUpdated == that.coordinationMetadataUpdated + && settingsMetadataUpdated == that.settingsMetadataUpdated + && transientSettingsMetadataUpdated == that.transientSettingsMetadataUpdated + && templatesMetadataUpdated == that.templatesMetadataUpdated + && clusterBlocksUpdated == that.clusterBlocksUpdated + && discoveryNodesUpdated == that.discoveryNodesUpdated + && hashesOfConsistentSettingsUpdated == that.hashesOfConsistentSettingsUpdated + && Objects.equals(fromStateUUID, that.fromStateUUID) + && Objects.equals(toStateUUID, that.toStateUUID) + && Objects.equals(customMetadataUpdated, that.customMetadataUpdated) + && Objects.equals(customMetadataDeleted, that.customMetadataDeleted) + && Objects.equals(indicesUpdated, that.indicesUpdated) + && Objects.equals(indicesDeleted, that.indicesDeleted) + && Objects.equals(indicesRoutingUpdated, that.indicesRoutingUpdated) + && Objects.equals(indicesRoutingDeleted, that.indicesRoutingDeleted) + && Objects.equals(clusterStateCustomUpdated, that.clusterStateCustomUpdated) + && Objects.equals(clusterStateCustomDeleted, that.clusterStateCustomDeleted); + } + + @Override + public int hashCode() { + return Objects.hash( + fromStateUUID, + toStateUUID, + coordinationMetadataUpdated, + settingsMetadataUpdated, + transientSettingsMetadataUpdated, + templatesMetadataUpdated, + customMetadataUpdated, + customMetadataDeleted, + indicesUpdated, + indicesDeleted, + clusterBlocksUpdated, + discoveryNodesUpdated, + indicesRoutingUpdated, + indicesRoutingDeleted, + hashesOfConsistentSettingsUpdated, + clusterStateCustomUpdated, + clusterStateCustomDeleted + ); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(fromStateUUID); + out.writeString(toStateUUID); + out.writeBoolean(coordinationMetadataUpdated); + out.writeBoolean(settingsMetadataUpdated); + out.writeBoolean(transientSettingsMetadataUpdated); + out.writeBoolean(templatesMetadataUpdated); + out.writeStringCollection(indicesUpdated); + out.writeStringCollection(indicesDeleted); + out.writeStringCollection(customMetadataUpdated); + out.writeStringCollection(customMetadataDeleted); + out.writeBoolean(clusterBlocksUpdated); + out.writeBoolean(discoveryNodesUpdated); + out.writeStringCollection(indicesRoutingUpdated); + out.writeStringCollection(indicesRoutingDeleted); + out.writeBoolean(hashesOfConsistentSettingsUpdated); + out.writeStringCollection(clusterStateCustomUpdated); + out.writeStringCollection(clusterStateCustomDeleted); + } + + /** + * Builder for ClusterStateDiffManifest + * + * @opensearch.internal + */ + public static class Builder { + private String fromStateUUID; + private String toStateUUID; + private boolean coordinationMetadataUpdated; + private boolean settingsMetadataUpdated; + private boolean transientSettingsMetadataUpdated; + private boolean templatesMetadataUpdated; + private List customMetadataUpdated; + private List customMetadataDeleted; + private List indicesUpdated; + private List indicesDeleted; + private boolean clusterBlocksUpdated; + private boolean discoveryNodesUpdated; + private List indicesRoutingUpdated; + private List indicesRoutingDeleted; + private boolean hashesOfConsistentSettingsUpdated; + private List clusterStateCustomUpdated; + private List clusterStateCustomDeleted; + + public Builder() {} + + public Builder fromStateUUID(String fromStateUUID) { + this.fromStateUUID = fromStateUUID; + return this; + } + + public Builder toStateUUID(String toStateUUID) { + this.toStateUUID = toStateUUID; + return this; + } + + public Builder coordinationMetadataUpdated(boolean coordinationMetadataUpdated) { + this.coordinationMetadataUpdated = coordinationMetadataUpdated; + return this; + } + + public Builder settingsMetadataUpdated(boolean settingsMetadataUpdated) { + this.settingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + + public Builder transientSettingsMetadataUpdate(boolean settingsMetadataUpdated) { + this.transientSettingsMetadataUpdated = settingsMetadataUpdated; + return this; + } + + public Builder templatesMetadataUpdated(boolean templatesMetadataUpdated) { + this.templatesMetadataUpdated = templatesMetadataUpdated; + return this; + } + + public Builder hashesOfConsistentSettingsUpdated(boolean hashesOfConsistentSettingsUpdated) { + this.hashesOfConsistentSettingsUpdated = hashesOfConsistentSettingsUpdated; + return this; + } + + public Builder customMetadataUpdated(List customMetadataUpdated) { + this.customMetadataUpdated = customMetadataUpdated; + return this; + } + + public Builder customMetadataDeleted(List customMetadataDeleted) { + this.customMetadataDeleted = customMetadataDeleted; + return this; + } + + public Builder indicesUpdated(List indicesUpdated) { + this.indicesUpdated = indicesUpdated; + return this; + } + + public Builder indicesDeleted(List indicesDeleted) { + this.indicesDeleted = indicesDeleted; + return this; + } + + public Builder clusterBlocksUpdated(boolean clusterBlocksUpdated) { + this.clusterBlocksUpdated = clusterBlocksUpdated; + return this; + } + + public Builder discoveryNodesUpdated(boolean discoveryNodesUpdated) { + this.discoveryNodesUpdated = discoveryNodesUpdated; + return this; + } + + public Builder indicesRoutingUpdated(List indicesRoutingUpdated) { + this.indicesRoutingUpdated = indicesRoutingUpdated; + return this; + } + + public Builder indicesRoutingDeleted(List indicesRoutingDeleted) { + this.indicesRoutingDeleted = indicesRoutingDeleted; + return this; + } + + public Builder clusterStateCustomUpdated(List clusterStateCustomUpdated) { + this.clusterStateCustomUpdated = clusterStateCustomUpdated; + return this; + } + + public Builder clusterStateCustomDeleted(List clusterStateCustomDeleted) { + this.clusterStateCustomDeleted = clusterStateCustomDeleted; + return this; + } + + public ClusterStateDiffManifest build() { + return new ClusterStateDiffManifest( + fromStateUUID, + toStateUUID, + coordinationMetadataUpdated, + settingsMetadataUpdated, + transientSettingsMetadataUpdated, + templatesMetadataUpdated, + customMetadataUpdated, + customMetadataDeleted, + indicesUpdated, + indicesDeleted, + clusterBlocksUpdated, + discoveryNodesUpdated, + indicesRoutingUpdated, + indicesRoutingDeleted, + hashesOfConsistentSettingsUpdated, + clusterStateCustomUpdated, + clusterStateCustomDeleted + ); + } } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java new file mode 100644 index 0000000000000..7e83a7bf7da44 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateAttributesManager.java @@ -0,0 +1,118 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.action.LatchedActionListener; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.CheckedRunnable; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.core.action.ActionListener; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.gateway.remote.model.RemoteClusterBlocks; +import org.opensearch.gateway.remote.model.RemoteClusterStateBlobStore; +import org.opensearch.gateway.remote.model.RemoteClusterStateCustoms; +import org.opensearch.gateway.remote.model.RemoteDiscoveryNodes; +import org.opensearch.gateway.remote.model.RemoteReadResult; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * A Manager which provides APIs to upload and download attributes of ClusterState to the {@link RemoteClusterStateBlobStore} + * + * @opensearch.internal + */ +public class RemoteClusterStateAttributesManager { + public static final String CLUSTER_STATE_ATTRIBUTE = "cluster_state_attribute"; + public static final String DISCOVERY_NODES = "nodes"; + public static final String CLUSTER_BLOCKS = "blocks"; + public static final int CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION = 1; + private final RemoteClusterStateBlobStore clusterBlocksBlobStore; + private final RemoteClusterStateBlobStore discoveryNodesBlobStore; + private final RemoteClusterStateBlobStore customsBlobStore; + private final Compressor compressor; + private final NamedXContentRegistry namedXContentRegistry; + private final NamedWriteableRegistry namedWriteableRegistry; + + RemoteClusterStateAttributesManager( + RemoteClusterStateBlobStore clusterBlocksBlobStore, + RemoteClusterStateBlobStore discoveryNodesBlobStore, + RemoteClusterStateBlobStore customsBlobStore, + Compressor compressor, + NamedXContentRegistry namedXContentRegistry, + NamedWriteableRegistry namedWriteableRegistry + ) { + this.clusterBlocksBlobStore = clusterBlocksBlobStore; + this.discoveryNodesBlobStore = discoveryNodesBlobStore; + this.customsBlobStore = customsBlobStore; + this.compressor = compressor; + this.namedXContentRegistry = namedXContentRegistry; + this.namedWriteableRegistry = namedWriteableRegistry; + } + + /** + * Allows async upload of Cluster State Attribute components to remote + */ + CheckedRunnable getAsyncMetadataWriteAction( + String component, + AbstractRemoteWritableBlobEntity blobEntity, + RemoteClusterStateBlobStore remoteEntityStore, + LatchedActionListener latchedActionListener + ) { + return () -> remoteEntityStore.writeAsync(blobEntity, getActionListener(component, blobEntity, latchedActionListener)); + } + + private ActionListener getActionListener( + String component, + AbstractRemoteWritableBlobEntity remoteObject, + LatchedActionListener latchedActionListener + ) { + return ActionListener.wrap( + resp -> latchedActionListener.onResponse(remoteObject.getUploadedMetadata()), + ex -> latchedActionListener.onFailure(new RemoteStateTransferException(component, remoteObject, ex)) + ); + } + + public CheckedRunnable getAsyncMetadataReadAction( + String component, + AbstractRemoteWritableBlobEntity blobEntity, + RemoteClusterStateBlobStore remoteEntityStore, + LatchedActionListener listener + ) { + final ActionListener actionListener = ActionListener.wrap( + response -> listener.onResponse(new RemoteReadResult((ToXContent) response, CLUSTER_STATE_ATTRIBUTE, component)), + listener::onFailure + ); + return () -> remoteEntityStore.readAsync(blobEntity, actionListener); + } + + public Map getUpdatedCustoms(ClusterState clusterState, ClusterState previousClusterState) { + Map updatedCustoms = new HashMap<>(); + Set currentCustoms = new HashSet<>(clusterState.customs().keySet()); + for (Map.Entry entry : previousClusterState.customs().entrySet()) { + if (currentCustoms.contains(entry.getKey()) && !entry.getValue().equals(clusterState.customs().get(entry.getKey()))) { + updatedCustoms.put(entry.getKey(), clusterState.customs().get(entry.getKey())); + } + currentCustoms.remove(entry.getKey()); + } + for (String custom : currentCustoms) { + updatedCustoms.put(custom, clusterState.customs().get(custom)); + } + return updatedCustoms; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java index 1617efd6200c3..f3c90239c7adb 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java @@ -75,6 +75,7 @@ import static java.util.Objects.requireNonNull; import static org.opensearch.gateway.PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled; /** @@ -171,12 +172,6 @@ public class RemoteClusterStateService implements Closeable { /** * Manifest format compatible with codec v2, where global metadata file is replaced with multiple metadata attribute files */ - public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT_V2 = - new ChecksumBlobStoreFormat<>("cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, ClusterMetadataManifest::fromXContentV2); - - /** - * Manifest format compatible with codec v3, where global metadata file is replaced with multiple metadata attribute files - */ public static final ChecksumBlobStoreFormat CLUSTER_METADATA_MANIFEST_FORMAT = new ChecksumBlobStoreFormat<>( "cluster-metadata-manifest", METADATA_MANIFEST_NAME_FORMAT, @@ -226,7 +221,6 @@ public class RemoteClusterStateService implements Closeable { + "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata " + "updated : [{}], custom metadata updated : [{}], indices routing updated : [{}]"; public static final int INDEX_METADATA_CURRENT_CODEC_VERSION = 1; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 2; // ToXContent Params with gateway mode. @@ -836,26 +830,25 @@ private RemoteClusterStateManifestInfo uploadManifest( committed, MANIFEST_CURRENT_CODEC_VERSION ); - final ClusterMetadataManifest manifest = new ClusterMetadataManifest( - clusterState.term(), - clusterState.getVersion(), - clusterState.metadata().clusterUUID(), - clusterState.stateUUID(), - Version.CURRENT, - nodeId, - committed, - MANIFEST_CURRENT_CODEC_VERSION, - null, - uploadedIndexMetadata, - previousClusterUUID, - clusterState.metadata().clusterUUIDCommitted(), - uploadedCoordinationMetadata, - uploadedSettingsMetadata, - uploadedTemplatesMetadata, - uploadedCustomMetadataMap, - clusterState.routingTable().version(), - uploadedIndicesRouting - ); + final ClusterMetadataManifest manifest = ClusterMetadataManifest.builder() + .clusterTerm(clusterState.term()) + .stateVersion(clusterState.getVersion()) + .clusterUUID(clusterState.metadata().clusterUUID()) + .stateUUID(clusterState.stateUUID()) + .opensearchVersion(Version.CURRENT) + .nodeId(nodeId) + .committed(committed) + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) + .indices(uploadedIndexMetadata) + .previousClusterUUID(previousClusterUUID) + .clusterUUIDCommitted(clusterState.metadata().clusterUUIDCommitted()) + .coordinationMetadata(uploadedCoordinationMetadata) + .settingMetadata(uploadedSettingsMetadata) + .templatesMetadata(uploadedTemplatesMetadata) + .customMetadataMap(uploadedCustomMetadataMap) + .routingTableVersion(clusterState.routingTable().version()) + .indicesRouting(uploadedIndicesRouting) + .build(); writeMetadataManifest(clusterState.getClusterName().value(), clusterState.metadata().clusterUUID(), manifest, manifestFileName); return new RemoteClusterStateManifestInfo(manifest, manifestFileName); } @@ -1540,8 +1533,6 @@ private ChecksumBlobStoreFormat getClusterMetadataManif long codecVersion = getManifestCodecVersion(fileName); if (codecVersion == MANIFEST_CURRENT_CODEC_VERSION) { return CLUSTER_METADATA_MANIFEST_FORMAT; - } else if (codecVersion == ClusterMetadataManifest.CODEC_V2) { - return CLUSTER_METADATA_MANIFEST_FORMAT_V2; } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { return CLUSTER_METADATA_MANIFEST_FORMAT_V1; } else if (codecVersion == ClusterMetadataManifest.CODEC_V0) { diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java index f87b0a6e401d3..8efde7ee45f49 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateUtils.java @@ -9,10 +9,17 @@ package org.opensearch.gateway.remote; import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.common.blobstore.BlobContainer; +import org.opensearch.common.blobstore.BlobPath; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; import java.util.Map; /** @@ -21,11 +28,14 @@ public class RemoteClusterStateUtils { public static final String DELIMITER = "__"; - public static final String PATH_DELIMITER = "/"; - public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; public static final String METADATA_NAME_FORMAT = "%s.dat"; - public static final String METADATA_NAME_PLAIN_FORMAT = "%s"; + public static final String CLUSTER_STATE_PATH_TOKEN = "cluster-state"; + public static final String GLOBAL_METADATA_PATH_TOKEN = "global-metadata"; + public static final String CLUSTER_STATE_EPHEMERAL_PATH_TOKEN = "ephemeral"; public static final int GLOBAL_METADATA_CURRENT_CODEC_VERSION = 1; + public static final String CUSTOM_DELIMITER = "--"; + public static final String PATH_DELIMITER = "/"; + public static final String METADATA_NAME_PLAIN_FORMAT = "%s"; // ToXContent Params with gateway mode. // We are using gateway context mode to persist all custom metadata. @@ -33,7 +43,84 @@ public class RemoteClusterStateUtils { Map.of(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY) ); + public static BlobPath getCusterMetadataBasePath(BlobStoreRepository blobStoreRepository, String clusterName, String clusterUUID) { + return blobStoreRepository.basePath().add(encodeString(clusterName)).add(CLUSTER_STATE_PATH_TOKEN).add(clusterUUID); + } + public static String encodeString(String content) { return Base64.getUrlEncoder().withoutPadding().encodeToString(content.getBytes(StandardCharsets.UTF_8)); } + + public static String getFormattedFileName(String fileName, int codecVersion) { + if (codecVersion < ClusterMetadataManifest.CODEC_V2) { + return String.format(Locale.ROOT, METADATA_NAME_FORMAT, fileName); + } + return fileName; + } + + static BlobContainer clusterUUIDContainer(BlobStoreRepository blobStoreRepository, String clusterName) { + return blobStoreRepository.blobStore() + .blobContainer( + blobStoreRepository.basePath() + .add(Base64.getUrlEncoder().withoutPadding().encodeToString(clusterName.getBytes(StandardCharsets.UTF_8))) + .add(CLUSTER_STATE_PATH_TOKEN) + ); + } + + /** + * Container class to keep metadata of all uploaded attributes + */ + public static class UploadedMetadataResults { + List uploadedIndexMetadata; + Map uploadedCustomMetadataMap; + Map uploadedClusterStateCustomMetadataMap; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks; + List uploadedIndicesRoutingMetadata; + ClusterMetadataManifest.UploadedMetadataAttribute uploadedHashesOfConsistentSettings; + + public UploadedMetadataResults( + List uploadedIndexMetadata, + Map uploadedCustomMetadataMap, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedCoordinationMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTransientSettingsMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedTemplatesMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedDiscoveryNodes, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedClusterBlocks, + List uploadedIndicesRoutingMetadata, + ClusterMetadataManifest.UploadedMetadataAttribute uploadedHashesOfConsistentSettings, + Map uploadedClusterStateCustomMap + ) { + this.uploadedIndexMetadata = uploadedIndexMetadata; + this.uploadedCustomMetadataMap = uploadedCustomMetadataMap; + this.uploadedCoordinationMetadata = uploadedCoordinationMetadata; + this.uploadedSettingsMetadata = uploadedSettingsMetadata; + this.uploadedTransientSettingsMetadata = uploadedTransientSettingsMetadata; + this.uploadedTemplatesMetadata = uploadedTemplatesMetadata; + this.uploadedDiscoveryNodes = uploadedDiscoveryNodes; + this.uploadedClusterBlocks = uploadedClusterBlocks; + this.uploadedIndicesRoutingMetadata = uploadedIndicesRoutingMetadata; + this.uploadedHashesOfConsistentSettings = uploadedHashesOfConsistentSettings; + this.uploadedClusterStateCustomMetadataMap = uploadedClusterStateCustomMap; + } + + public UploadedMetadataResults() { + this.uploadedIndexMetadata = new ArrayList<>(); + this.uploadedCustomMetadataMap = new HashMap<>(); + this.uploadedCoordinationMetadata = null; + this.uploadedSettingsMetadata = null; + this.uploadedTransientSettingsMetadata = null; + this.uploadedTemplatesMetadata = null; + this.uploadedDiscoveryNodes = null; + this.uploadedClusterBlocks = null; + this.uploadedIndicesRoutingMetadata = new ArrayList<>(); + this.uploadedHashesOfConsistentSettings = null; + this.uploadedClusterStateCustomMetadataMap = new HashMap<>(); + } + } } diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java new file mode 100644 index 0000000000000..5b75b6c18ee5b --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteStateTransferException.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote; + +import org.opensearch.common.remote.RemoteWriteableEntity; + +/** + * Exception for Remote state transfer. + */ +public class RemoteStateTransferException extends RuntimeException { + private RemoteWriteableEntity entity; + + public RemoteStateTransferException(String errorDesc) { + super(errorDesc); + } + + public RemoteStateTransferException(String errorDesc, Throwable cause) { + super(errorDesc, cause); + } + + public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity) { + super(errorDesc); + this.entity = entity; + } + + public RemoteStateTransferException(String errorDesc, RemoteWriteableEntity entity, Throwable cause) { + super(errorDesc, cause); + this.entity = entity; + } + + @Override + public String toString() { + String message = super.toString(); + if (entity != null) { + message += ", failed entity:" + entity; + } + return message; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java new file mode 100644 index 0000000000000..937f9dc2c8631 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterBlocks.java @@ -0,0 +1,96 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.compress.Compressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.core.common.bytes.BytesReference.toBytes; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; + +/** + * Wrapper class for uploading/downloading {@link ClusterBlocks} to/from remote blob store + */ +public class RemoteClusterBlocks extends AbstractRemoteWritableBlobEntity { + + public static final String CLUSTER_BLOCKS = "blocks"; + + private ClusterBlocks clusterBlocks; + private long stateVersion; + + public RemoteClusterBlocks(final ClusterBlocks clusterBlocks, long stateVersion, String clusterUUID, final Compressor compressor) { + super(clusterUUID, compressor, null); + this.clusterBlocks = clusterBlocks; + this.stateVersion = stateVersion; + } + + public RemoteClusterBlocks(final String blobName, final String clusterUUID, final Compressor compressor) { + super(clusterUUID, compressor, null); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), CLUSTER_BLOCKS); + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/transient/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(CLUSTER_BLOCKS, blobName); + } + + @Override + public InputStream serialize() throws IOException { + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { + clusterBlocks.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize remote cluster blocks", e); + } + } + + @Override + public ClusterBlocks deserialize(final InputStream inputStream) throws IOException { + try (StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream)))) { + return ClusterBlocks.readFrom(in); + } catch (IOException e) { + throw new IOException("Failed to deserialize remote cluster blocks", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java index 62bc4ce022a37..f7024c8fde7ed 100644 --- a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java @@ -35,7 +35,7 @@ public class RemoteClusterMetadataManifest extends AbstractRemoteWritableBlobEnt public static final int SPLITTED_MANIFEST_FILE_LENGTH = 6; public static final String METADATA_MANIFEST_NAME_FORMAT = "%s"; - public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V3; + public static final int MANIFEST_CURRENT_CODEC_VERSION = ClusterMetadataManifest.CODEC_V2; public static final String COMMITTED = "C"; public static final String PUBLISHED = "P"; diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java new file mode 100644 index 0000000000000..60a21c9b53148 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustoms.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.compress.Compressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.index.remote.RemoteStoreUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.cluster.ClusterState.FeatureAware.shouldSerialize; +import static org.opensearch.core.common.bytes.BytesReference.toBytes; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; + +/** + * Wrapper class for uploading/downloading {@link Custom} to/from remote blob store + */ +public class RemoteClusterStateCustoms extends AbstractRemoteWritableBlobEntity { + public static final String CLUSTER_STATE_CUSTOM = "cluster-state-custom"; + + private long stateVersion; + private String customType; + private ClusterState.Custom custom; + private final NamedWriteableRegistry namedWriteableRegistry; + + public RemoteClusterStateCustoms( + final ClusterState.Custom custom, + final String customType, + final long stateVersion, + final String clusterUUID, + final Compressor compressor, + final NamedWriteableRegistry namedWriteableRegistry + ) { + super(clusterUUID, compressor, null); + this.stateVersion = stateVersion; + this.customType = customType; + this.custom = custom; + this.namedWriteableRegistry = namedWriteableRegistry; + } + + public RemoteClusterStateCustoms( + final String blobName, + final String customType, + final String clusterUUID, + final Compressor compressor, + final NamedWriteableRegistry namedWriteableRegistry + ) { + super(clusterUUID, compressor, null); + this.blobName = blobName; + this.customType = customType; + this.namedWriteableRegistry = namedWriteableRegistry; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + String prefix = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType); + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), prefix); + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new ClusterMetadataManifest.UploadedMetadataAttribute( + String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, customType), + blobName + ); + } + + @Override + public InputStream serialize() throws IOException { + try (BytesStreamOutput outputStream = new BytesStreamOutput()) { + if (shouldSerialize(outputStream, custom)) { + outputStream.writeNamedWriteable(custom); + } + return outputStream.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize cluster state custom of type " + customType, e); + } + } + + @Override + public ClusterState.Custom deserialize(final InputStream inputStream) throws IOException { + try ( + NamedWriteableAwareStreamInput in = new NamedWriteableAwareStreamInput( + new BytesStreamInput(toBytes(Streams.readFully(inputStream))), + this.namedWriteableRegistry + ) + ) { + return in.readNamedWriteable(Custom.class); + } catch (IOException e) { + throw new IOException("Failed to deserialize cluster state custom of type " + customType, e); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java new file mode 100644 index 0000000000000..7dc2b6492de7e --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodes.java @@ -0,0 +1,101 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.compress.Compressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.index.remote.RemoteStoreUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.core.common.bytes.BytesReference.toBytes; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; + +/** + * Wrapper class for uploading/downloading {@link DiscoveryNodes} to/from remote blob store + */ +public class RemoteDiscoveryNodes extends AbstractRemoteWritableBlobEntity { + + public static final String DISCOVERY_NODES = "nodes"; + + private DiscoveryNodes discoveryNodes; + private long stateVersion; + + public RemoteDiscoveryNodes( + final DiscoveryNodes discoveryNodes, + final long stateVersion, + final String clusterUUID, + final Compressor compressor + ) { + super(clusterUUID, compressor, null); + this.discoveryNodes = discoveryNodes; + this.stateVersion = stateVersion; + } + + public RemoteDiscoveryNodes(final String blobName, final String clusterUUID, final Compressor compressor) { + super(clusterUUID, compressor, null); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN), DISCOVERY_NODES); + } + + @Override + public String generateBlobFileName() { + // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/ephemeral/______ + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(stateVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(DISCOVERY_NODES, blobName); + } + + @Override + public InputStream serialize() throws IOException { + try (BytesStreamOutput outputStream = new BytesStreamOutput()) { + discoveryNodes.writeTo(outputStream); + return outputStream.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize remote discovery nodes", e); + } + } + + @Override + public DiscoveryNodes deserialize(final InputStream inputStream) throws IOException { + try (StreamInput streamInput = new BytesStreamInput(toBytes(Streams.readFully(inputStream)))) { + return DiscoveryNodes.readFrom(streamInput, null); + } catch (IOException e) { + throw new IOException("Failed to deserialize remote discovery nodes", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java new file mode 100644 index 0000000000000..a70506bcd6846 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettings.java @@ -0,0 +1,99 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.io.Streams; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.common.io.stream.BytesStreamInput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.compress.Compressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.index.remote.RemoteStoreUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.core.common.bytes.BytesReference.toBytes; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; + +/** + * Wrapper class for uploading/downloading {@link DiffableStringMap} to/from remote blob store + */ +public class RemoteHashesOfConsistentSettings extends AbstractRemoteWritableBlobEntity { + public static final String HASHES_OF_CONSISTENT_SETTINGS = "hashes-of-consistent-settings"; + + private DiffableStringMap hashesOfConsistentSettings; + private long metadataVersion; + + public RemoteHashesOfConsistentSettings( + final DiffableStringMap hashesOfConsistentSettings, + final long metadataVersion, + final String clusterUUID, + final Compressor compressor + ) { + super(clusterUUID, compressor, null); + this.metadataVersion = metadataVersion; + this.hashesOfConsistentSettings = hashesOfConsistentSettings; + } + + public RemoteHashesOfConsistentSettings(final String blobName, final String clusterUUID, final Compressor compressor) { + super(clusterUUID, compressor, null); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), HASHES_OF_CONSISTENT_SETTINGS); + } + + @Override + public String generateBlobFileName() { + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public ClusterMetadataManifest.UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new ClusterMetadataManifest.UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, blobName); + } + + @Override + public InputStream serialize() throws IOException { + try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) { + hashesOfConsistentSettings.writeTo(bytesStreamOutput); + return bytesStreamOutput.bytes().streamInput(); + } catch (IOException e) { + throw new IOException("Failed to serialize hashes of consistent settings", e); + } + + } + + @Override + public DiffableStringMap deserialize(final InputStream inputStream) throws IOException { + try (StreamInput in = new BytesStreamInput(toBytes(Streams.readFully(inputStream)))) { + return DiffableStringMap.readFrom(in); + } catch (IOException e) { + throw new IOException("Failed to deserialize hashes of consistent settings", e); + } + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java new file mode 100644 index 0000000000000..adee09eaeffef --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteReadResult.java @@ -0,0 +1,39 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.core.xcontent.ToXContent; + +/** + * Container class for entity read from remote store + */ +public class RemoteReadResult { + + ToXContent obj; + String component; + String componentName; + + public RemoteReadResult(ToXContent obj, String component, String componentName) { + this.obj = obj; + this.component = component; + this.componentName = componentName; + } + + public ToXContent getObj() { + return obj; + } + + public String getComponent() { + return component; + } + + public String getComponentName() { + return componentName; + } +} diff --git a/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java new file mode 100644 index 0000000000000..fe32b95f5e957 --- /dev/null +++ b/server/src/main/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadata.java @@ -0,0 +1,108 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.common.io.Streams; +import org.opensearch.common.remote.AbstractRemoteWritableBlobEntity; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.METADATA_NAME_FORMAT; + +/** + * Wrapper class for uploading/downloading transient {@link Settings} to/from remote blob store + */ +public class RemoteTransientSettingsMetadata extends AbstractRemoteWritableBlobEntity { + + public static final String TRANSIENT_SETTING_METADATA = "transient-settings"; + + public static final ChecksumBlobStoreFormat SETTINGS_METADATA_FORMAT = new ChecksumBlobStoreFormat<>( + "transient-settings", + METADATA_NAME_FORMAT, + Settings::fromXContent + ); + + private Settings transientSettings; + private long metadataVersion; + + public RemoteTransientSettingsMetadata( + final Settings transientSettings, + final long metadataVersion, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { + super(clusterUUID, compressor, namedXContentRegistry); + this.transientSettings = transientSettings; + this.metadataVersion = metadataVersion; + } + + public RemoteTransientSettingsMetadata( + final String blobName, + final String clusterUUID, + final Compressor compressor, + final NamedXContentRegistry namedXContentRegistry + ) { + super(clusterUUID, compressor, namedXContentRegistry); + this.blobName = blobName; + } + + @Override + public BlobPathParameters getBlobPathParameters() { + return new BlobPathParameters(List.of(GLOBAL_METADATA_PATH_TOKEN), TRANSIENT_SETTING_METADATA); + } + + @Override + public String generateBlobFileName() { + String blobFileName = String.join( + DELIMITER, + getBlobPathParameters().getFilePrefix(), + RemoteStoreUtils.invertLong(metadataVersion), + RemoteStoreUtils.invertLong(System.currentTimeMillis()), + String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION) + ); + this.blobFileName = blobFileName; + return blobFileName; + } + + @Override + public InputStream serialize() throws IOException { + return SETTINGS_METADATA_FORMAT.serialize( + transientSettings, + generateBlobFileName(), + getCompressor(), + RemoteClusterStateUtils.FORMAT_PARAMS + ).streamInput(); + } + + @Override + public Settings deserialize(final InputStream inputStream) throws IOException { + return SETTINGS_METADATA_FORMAT.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream)); + } + + @Override + public UploadedMetadata getUploadedMetadata() { + assert blobName != null; + return new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, blobName); + } +} diff --git a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java index 04e04bd96a7d3..0f049f99b49c0 100644 --- a/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java +++ b/server/src/test/java/org/opensearch/cluster/block/ClusterBlockTests.java @@ -136,7 +136,7 @@ public void testGetIndexBlockWithId() { assertThat(builder.build().getIndexBlockWithId("index", randomValueOtherThan(blockId, OpenSearchTestCase::randomInt)), nullValue()); } - private ClusterBlock randomClusterBlock() { + public static ClusterBlock randomClusterBlock() { final String uuid = randomBoolean() ? UUIDs.randomBase64UUID() : null; final List levels = Arrays.asList(ClusterBlockLevel.values()); return new ClusterBlock( diff --git a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java index 7cdadfe9e96c3..6da8cbff2c688 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/ClusterMetadataManifestTests.java @@ -9,6 +9,7 @@ package org.opensearch.gateway.remote; import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexGraveyard; import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.metadata.WeightedRoutingMetadata; @@ -34,6 +35,11 @@ import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0; import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_BLOCKS; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.DISCOVERY_NODES; +import static org.opensearch.gateway.remote.RemoteClusterStateServiceTests.generateClusterStateWithOneIndex; +import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; +import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; public class ClusterMetadataManifestTests extends OpenSearchTestCase { @@ -92,42 +98,43 @@ public void testClusterMetadataManifestXContentV1() throws IOException { public void testClusterMetadataManifestXContent() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V3, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V2) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file")) + .settingMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file")) + .templatesMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file")) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 1L, - randomUploadedIndexMetadataList() - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -140,42 +147,49 @@ public void testClusterMetadataManifestXContent() throws IOException { } public void testClusterMetadataManifestSerializationEqualsHashCode() { - ClusterMetadataManifest initialManifest = new ClusterMetadataManifest( - 1337L, - 7L, - "HrYF3kP5SmSPWtKlWhnNSA", - "6By9p9G0Rv2MmFYJcPAOgA", - Version.CURRENT, - "B10RX1f5RJenMQvYccCgSQ", - true, - 2, - null, - randomUploadedIndexMetadataList(), - "yfObdx8KSMKKrXf8UyHhM", - true, - new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file"), - new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file"), - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest initialManifest = ClusterMetadataManifest.builder() + .clusterTerm(1337L) + .stateVersion(7L) + .clusterUUID("HrYF3kP5SmSPWtKlWhnNSA") + .stateUUID("6By9p9G0Rv2MmFYJcPAOgA") + .opensearchVersion(Version.CURRENT) + .nodeId("B10RX1f5RJenMQvYccCgSQ") + .committed(true) + .codecVersion(ClusterMetadataManifest.CODEC_V2) + .indices(randomUploadedIndexMetadataList()) + .previousClusterUUID("yfObdx8KSMKKrXf8UyHhM") + .clusterUUIDCommitted(true) + .coordinationMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.COORDINATION_METADATA, "coordination-file")) + .settingMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.SETTING_METADATA, "setting-file")) + .templatesMetadata(new UploadedMetadataAttribute(RemoteClusterStateService.TEMPLATES_METADATA, "templates-file")) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 1L, - randomUploadedIndexMetadataList() - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .discoveryNodesMetadata(new UploadedMetadataAttribute(DISCOVERY_NODES, "discovery-nodes-file")) + .clusterBlocksMetadata(new UploadedMetadataAttribute(CLUSTER_BLOCKS, "cluster-block-file")) + .transientSettingsMetadata(new UploadedMetadataAttribute(TRANSIENT_SETTING_METADATA, "transient-settings-file")) + .hashesOfConsistentSettings(new UploadedMetadataAttribute(HASHES_OF_CONSISTENT_SETTINGS, "hashes-of-consistent-settings-file")) + .clusterStateCustomMetadataMap(Collections.emptyMap()) + .diffManifest(new ClusterStateDiffManifest(generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE)) + .build(); { // Mutate Cluster Term EqualsHashCodeTestUtils.checkEqualsAndHashCode( initialManifest, @@ -312,59 +326,211 @@ public void testClusterMetadataManifestSerializationEqualsHashCode() { } ); } + { + // Mutate Coordination metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.coordinationMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate setting metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.settingMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate template metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.templatesMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate custom metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.customMetadataMap(Collections.emptyMap()); + return builder.build(); + } + ); + } + { + // Mutate discovery nodes + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.discoveryNodesMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate cluster blocks + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.clusterBlocksMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate transient settings metadata + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.transientSettingsMetadata(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } + { + // Mutate diff manifest + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.diffManifest(null); + return builder.build(); + } + ); + } + { + // Mutate hashes of consistent settings + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + initialManifest, + orig -> OpenSearchTestCase.copyWriteable( + orig, + new NamedWriteableRegistry(Collections.emptyList()), + ClusterMetadataManifest::new + ), + manifest -> { + ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder(manifest); + builder.hashesOfConsistentSettings(randomUploadedMetadataAttribute()); + return builder.build(); + } + ); + } } public void testClusterMetadataManifestXContentV2() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V2, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V2) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(uploadedMetadataAttribute) + .settingMetadata(uploadedMetadataAttribute) + .templatesMetadata(uploadedMetadataAttribute) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 0, - new ArrayList<>() - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .routingTableVersion(1L) + .indicesRouting(Collections.singletonList(uploadedIndexMetadata)) + .discoveryNodesMetadata(uploadedMetadataAttribute) + .clusterBlocksMetadata(uploadedMetadataAttribute) + .transientSettingsMetadata(uploadedMetadataAttribute) + .hashesOfConsistentSettings(uploadedMetadataAttribute) + .clusterStateCustomMetadataMap(Collections.emptyMap()) + .diffManifest(new ClusterStateDiffManifest(generateClusterStateWithOneIndex().build(), ClusterState.EMPTY_STATE)) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); builder.endObject(); try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { - final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContentV2(parser); + final ClusterMetadataManifest fromXContentManifest = ClusterMetadataManifest.fromXContent(parser); assertEquals(originalManifest, fromXContentManifest); } } - public void testClusterMetadataManifestXContentV3() throws IOException { + public void testClusterMetadataManifestXContentV2WithoutEphemeral() throws IOException { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); UploadedMetadataAttribute uploadedMetadataAttribute = new UploadedMetadataAttribute("attribute_name", "testing_attribute"); UploadedIndexMetadata uploadedIndexRoutingMetadata = new UploadedIndexMetadata( @@ -373,43 +539,43 @@ public void testClusterMetadataManifestXContentV3() throws IOException { "routing-path", InternalRemoteRoutingTableService.INDEX_ROUTING_METADATA_PREFIX ); - - ClusterMetadataManifest originalManifest = new ClusterMetadataManifest( - 1L, - 1L, - "test-cluster-uuid", - "test-state-uuid", - Version.CURRENT, - "test-node-id", - false, - ClusterMetadataManifest.CODEC_V3, - null, - Collections.singletonList(uploadedIndexMetadata), - "prev-cluster-uuid", - true, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - uploadedMetadataAttribute, - Collections.unmodifiableList( - Arrays.asList( - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + RepositoriesMetadata.TYPE, - "custom--repositories-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, - "custom--index_graveyard-file" - ), - new UploadedMetadataAttribute( - RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER - + WeightedRoutingMetadata.TYPE, - "custom--weighted_routing_netadata-file" + ClusterMetadataManifest originalManifest = ClusterMetadataManifest.builder() + .clusterTerm(1L) + .stateVersion(1L) + .clusterUUID("test-cluster-uuid") + .stateUUID("test-state-uuid") + .opensearchVersion(Version.CURRENT) + .nodeId("test-node-id") + .committed(false) + .codecVersion(ClusterMetadataManifest.CODEC_V2) + .indices(Collections.singletonList(uploadedIndexMetadata)) + .previousClusterUUID("prev-cluster-uuid") + .clusterUUIDCommitted(true) + .coordinationMetadata(uploadedMetadataAttribute) + .settingMetadata(uploadedMetadataAttribute) + .templatesMetadata(uploadedMetadataAttribute) + .customMetadataMap( + Collections.unmodifiableList( + Arrays.asList( + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + RepositoriesMetadata.TYPE, + "custom--repositories-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + IndexGraveyard.TYPE, + "custom--index_graveyard-file" + ), + new UploadedMetadataAttribute( + RemoteClusterStateService.CUSTOM_METADATA + RemoteClusterStateService.CUSTOM_DELIMITER + + WeightedRoutingMetadata.TYPE, + "custom--weighted_routing_netadata-file" + ) ) - ) - ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())), - 1L, - Collections.singletonList(uploadedIndexRoutingMetadata) - ); + ).stream().collect(Collectors.toMap(UploadedMetadataAttribute::getAttributeName, Function.identity())) + ) + .indicesRouting(Collections.singletonList(uploadedIndexRoutingMetadata)) + .build(); final XContentBuilder builder = JsonXContent.contentBuilder(); builder.startObject(); originalManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); @@ -434,6 +600,10 @@ private UploadedIndexMetadata randomUploadedIndexMetadata() { return new UploadedIndexMetadata(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10)); } + private UploadedMetadataAttribute randomUploadedMetadataAttribute() { + return new UploadedMetadataAttribute("attribute_name", "testing_attribute"); + } + public void testUploadedIndexMetadataSerializationEqualsHashCode() { UploadedIndexMetadata uploadedIndexMetadata = new UploadedIndexMetadata("test-index", "test-uuid", "/test/upload/path"); EqualsHashCodeTestUtils.checkEqualsAndHashCode( diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java index e91eeb82d44b9..7e10442d011fa 100644 --- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java +++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java @@ -90,15 +90,16 @@ import static java.util.stream.Collectors.toList; import static org.opensearch.common.util.FeatureFlags.REMOTE_PUBLICATION_EXPERIMENTAL; +import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1; import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER; import static org.opensearch.gateway.remote.RemoteClusterStateService.FORMAT_PARAMS; import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_CURRENT_CODEC_VERSION; -import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.gateway.remote.RemoteClusterStateService.MANIFEST_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX; import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA; import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA; +import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; @@ -490,7 +491,7 @@ public void testMigrationFromCodecV0ManifestToCodecV2Manifest() throws IOExcepti * even if it was not changed in this cluster state update */ public void testMigrationFromCodecV1ManifestToCodecV2Manifest() throws IOException { - verifyCodecMigrationManifest(ClusterMetadataManifest.CODEC_V1); + verifyCodecMigrationManifest(CODEC_V1); } private void verifyCodecMigrationManifest(int previousCodec) throws IOException { @@ -539,7 +540,7 @@ public void testWriteIncrementalGlobalMetadataFromCodecV0Success() throws IOExce public void testWriteIncrementalGlobalMetadataFromCodecV1Success() throws IOException { final ClusterMetadataManifest previousManifest = ClusterMetadataManifest.builder() - .codecVersion(1) + .codecVersion(CODEC_V1) .globalMetadataFileName("global-metadata-file") .indices(Collections.emptyList()) .build(); @@ -563,7 +564,7 @@ private void verifyWriteIncrementalGlobalMetadataFromOlderCodecSuccess(ClusterMe ).getClusterMetadataManifest(); final ClusterMetadataManifest expectedManifest = ClusterMetadataManifest.builder() - .codecVersion(3) + .codecVersion(MANIFEST_CURRENT_CODEC_VERSION) .indices(Collections.emptyList()) .clusterTerm(1L) .stateVersion(1L) @@ -1126,7 +1127,7 @@ public void testReadGlobalMetadataIOException() throws IOException { .stateVersion(1L) .stateUUID("state-uuid") .clusterUUID("cluster-uuid") - .codecVersion(ClusterMetadataManifest.CODEC_V1) + .codecVersion(CODEC_V1) .globalMetadataFileName(globalIndexMetadataName) .nodeId("nodeA") .opensearchVersion(VersionUtils.randomOpenSearchVersion(random())) @@ -1337,7 +1338,7 @@ public void testFileNames() { assertThat(splittedIndexMetadataFileName[3], is(String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION))); verifyManifestFileNameWithCodec(MANIFEST_CURRENT_CODEC_VERSION); - verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V1); + verifyManifestFileNameWithCodec(CODEC_V1); verifyManifestFileNameWithCodec(ClusterMetadataManifest.CODEC_V0); } @@ -1659,7 +1660,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap1 = Map.of("index-uuid1", indexMetadata1, "index-uuid2", indexMetadata2); mockBlobContainerForGlobalMetadata(blobContainer1, clusterManifest1, metadata1); - mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V3); + mockBlobContainer(blobContainer1, clusterManifest1, indexMetadataMap1, ClusterMetadataManifest.CODEC_V2); List uploadedIndexMetadataList2 = List.of( new UploadedIndexMetadata("index1", "index-uuid1", "key1"), @@ -1691,7 +1692,7 @@ private void mockObjectsForGettingPreviousClusterUUID( .build(); Map indexMetadataMap2 = Map.of("index-uuid1", indexMetadata3, "index-uuid2", indexMetadata4); mockBlobContainerForGlobalMetadata(blobContainer2, clusterManifest2, metadata2); - mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V3); + mockBlobContainer(blobContainer2, clusterManifest2, indexMetadataMap2, ClusterMetadataManifest.CODEC_V2); // differGlobalMetadata controls which one of IndexMetadata or Metadata object would be different // when comparing cluster-uuid3 and cluster-uuid1 state. @@ -1725,7 +1726,7 @@ private void mockObjectsForGettingPreviousClusterUUID( clusterUUIDCommitted.getOrDefault("cluster-uuid3", true) ); mockBlobContainerForGlobalMetadata(blobContainer3, clusterManifest3, metadata3); - mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V3); + mockBlobContainer(blobContainer3, clusterManifest3, indexMetadataMap3, ClusterMetadataManifest.CODEC_V2); ArrayList mockBlobContainerOrderedList = new ArrayList<>( List.of(blobContainer1, blobContainer1, blobContainer3, blobContainer3, blobContainer2, blobContainer2) @@ -1765,7 +1766,7 @@ private ClusterMetadataManifest generateV1ClusterMetadataManifest( .committed(true) .clusterUUIDCommitted(isUUIDCommitted) .globalMetadataFileName(globalMetadataFileName) - .codecVersion(ClusterMetadataManifest.CODEC_V1) + .codecVersion(CODEC_V1) .build(); } @@ -1837,7 +1838,7 @@ private void mockBlobContainer( Map indexMetadataMap, int codecVersion ) throws IOException { - String manifestFileName = codecVersion >= ClusterMetadataManifest.CODEC_V1 + String manifestFileName = codecVersion >= CODEC_V1 ? "manifest__manifestFileName__abcd__abcd__abcd__" + codecVersion : "manifestFileName"; BlobMetadata blobMetadata = new PlainBlobMetadata(manifestFileName, 1); @@ -1957,7 +1958,7 @@ private void mockBlobContainerForGlobalMetadata( } ); } - } else if (codecVersion == ClusterMetadataManifest.CODEC_V1) { + } else if (codecVersion == CODEC_V1) { String[] splitPath = clusterMetadataManifest.getGlobalMetadataFileName().split("/"); when(blobContainer.readBlob(RemoteClusterStateService.GLOBAL_METADATA_FORMAT.blobName(splitPath[splitPath.length - 1]))) .thenAnswer((invocationOnMock) -> { diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java new file mode 100644 index 0000000000000..897b2f5eeb25d --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/ClusterStateDiffManifestTests.java @@ -0,0 +1,210 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterState; +import org.opensearch.cluster.coordination.CoordinationMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.IndexTemplateMetadata; +import org.opensearch.cluster.metadata.Metadata; +import org.opensearch.cluster.metadata.TemplatesMetadata; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.transport.TransportAddress; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.gateway.remote.ClusterStateDiffManifest; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static org.opensearch.Version.CURRENT; +import static org.opensearch.cluster.ClusterState.EMPTY_STATE; +import static org.opensearch.core.common.transport.TransportAddress.META_ADDRESS; +import static org.opensearch.gateway.remote.model.RemoteClusterBlocksTests.randomClusterBlocks; + +public class ClusterStateDiffManifestTests extends OpenSearchTestCase { + + public void testClusterStateDiffManifest() { + ClusterState initialState = ClusterState.builder(EMPTY_STATE) + .metadata( + Metadata.builder() + .put( + IndexMetadata.builder("index-1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + ) + ) + .build(); + updateAndVerifyState( + initialState, + singletonList( + IndexMetadata.builder("index-2") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .build() + ), + singletonList("index-1"), + emptyMap(), + emptyList(), + emptyMap(), + emptyList(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean() + ); + } + + public void testClusterStateDiffManifestXContent() throws IOException { + ClusterState initialState = ClusterState.builder(EMPTY_STATE) + .metadata( + Metadata.builder() + .put( + IndexMetadata.builder("index-1") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + ) + ) + .build(); + ClusterStateDiffManifest diffManifest = updateAndVerifyState( + initialState, + emptyList(), + singletonList("index-1"), + emptyMap(), + emptyList(), + emptyMap(), + emptyList(), + true, + true, + true, + true, + true, + true, + true + ); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + diffManifest.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, BytesReference.bytes(builder))) { + final ClusterStateDiffManifest parsedManifest = ClusterStateDiffManifest.fromXContent(parser); + assertEquals(diffManifest, parsedManifest); + } + } + + private ClusterStateDiffManifest updateAndVerifyState( + ClusterState initialState, + List indicesToAdd, + List indicesToRemove, + Map customsToAdd, + List customsToRemove, + Map clusterStateCustomsToAdd, + List clusterStateCustomsToRemove, + boolean updateCoordinationState, + boolean updatePersistentSettings, + boolean updateTemplates, + boolean updateTransientSettings, + boolean updateDiscoveryNodes, + boolean updateClusterBlocks, + boolean updateHashesOfConsistentSettings + ) { + ClusterState.Builder clusterStateBuilder = ClusterState.builder(initialState); + Metadata.Builder metadataBuilder = Metadata.builder(initialState.metadata()); + for (IndexMetadata indexMetadata : indicesToAdd) { + metadataBuilder.put(indexMetadata, true); + } + indicesToRemove.forEach(metadataBuilder::remove); + for (String custom : customsToAdd.keySet()) { + metadataBuilder.putCustom(custom, customsToAdd.get(custom)); + } + customsToRemove.forEach(metadataBuilder::removeCustom); + for (String custom : clusterStateCustomsToAdd.keySet()) { + clusterStateBuilder.putCustom(custom, clusterStateCustomsToAdd.get(custom)); + } + clusterStateCustomsToRemove.forEach(clusterStateBuilder::removeCustom); + if (updateCoordinationState) { + metadataBuilder.coordinationMetadata( + CoordinationMetadata.builder(initialState.metadata().coordinationMetadata()) + .addVotingConfigExclusion(new CoordinationMetadata.VotingConfigExclusion("exlucdedNodeId", "excludedNodeName")) + .build() + ); + } + if (updatePersistentSettings) { + metadataBuilder.persistentSettings(Settings.builder().put("key", "value").build()); + } + if (updateTemplates) { + metadataBuilder.templates( + TemplatesMetadata.builder() + .put( + IndexTemplateMetadata.builder("template" + randomAlphaOfLength(3)) + .patterns(asList("bar-*", "foo-*")) + .settings( + Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build() + ) + .build() + ) + .build() + ); + } + if (updateTransientSettings) { + metadataBuilder.transientSettings(Settings.builder().put("key", "value").build()); + } + if (updateDiscoveryNodes) { + clusterStateBuilder.nodes( + DiscoveryNodes.builder(initialState.nodes()) + .add(new DiscoveryNode("new-cluster-manager", new TransportAddress(META_ADDRESS, 9200), CURRENT)) + .clusterManagerNodeId("new-cluster-manager") + ); + } + if (updateHashesOfConsistentSettings) { + metadataBuilder.hashesOfConsistentSettings(Collections.singletonMap("key", "value")); + } + if (updateClusterBlocks) { + clusterStateBuilder.blocks(randomClusterBlocks()); + } + ClusterState updatedClusterState = clusterStateBuilder.metadata(metadataBuilder.build()).build(); + + ClusterStateDiffManifest manifest = new ClusterStateDiffManifest(updatedClusterState, initialState); + assertEquals(indicesToAdd.stream().map(im -> im.getIndex().getName()).collect(toList()), manifest.getIndicesUpdated()); + assertEquals(indicesToRemove, manifest.getIndicesDeleted()); + assertEquals(new ArrayList<>(customsToAdd.keySet()), manifest.getCustomMetadataUpdated()); + assertEquals(customsToRemove, manifest.getCustomMetadataDeleted()); + assertEquals(new ArrayList<>(clusterStateCustomsToAdd.keySet()), manifest.getClusterStateCustomUpdated()); + assertEquals(clusterStateCustomsToRemove, manifest.getClusterStateCustomDeleted()); + assertEquals(updateCoordinationState, manifest.isCoordinationMetadataUpdated()); + assertEquals(updatePersistentSettings, manifest.isSettingsMetadataUpdated()); + assertEquals(updateTemplates, manifest.isTemplatesMetadataUpdated()); + assertEquals(updateTransientSettings, manifest.isTransientSettingsMetadataUpdated()); + assertEquals(updateDiscoveryNodes, manifest.isDiscoveryNodesUpdated()); + assertEquals(updateClusterBlocks, manifest.isClusterBlocksUpdated()); + assertEquals(updateHashesOfConsistentSettings, manifest.isHashesOfConsistentSettingsUpdated()); + return manifest; + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java new file mode 100644 index 0000000000000..a5419a8cc8115 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterBlocksTests.java @@ -0,0 +1,155 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.block.ClusterBlocks; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.cluster.block.ClusterBlockTests.randomClusterBlock; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteClusterBlocks.CLUSTER_BLOCKS; + +public class RemoteClusterBlocksTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/cluster-blocks"; + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(uploadedFile, clusterUUID, compressor); + assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "cluster-blocks" }); + } + + public void testBlobPathParameters() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), CLUSTER_BLOCKS); + } + + public void testGenerateBlobFileName() { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], CLUSTER_BLOCKS); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION)); + + } + + public void testGetUploadedMetadata() throws IOException { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), CLUSTER_BLOCKS); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + + public void testSerDe() throws IOException { + ClusterBlocks clusterBlocks = randomClusterBlocks(); + RemoteClusterBlocks remoteObjectForUpload = new RemoteClusterBlocks(clusterBlocks, METADATA_VERSION, clusterUUID, compressor); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + ClusterBlocks readClusterBlocks = remoteObjectForUpload.deserialize(inputStream); + assertEquals(clusterBlocks.global(), readClusterBlocks.global()); + assertEquals(clusterBlocks.indices().keySet(), readClusterBlocks.indices().keySet()); + for (String index : clusterBlocks.indices().keySet()) { + assertEquals(clusterBlocks.indices().get(index), readClusterBlocks.indices().get(index)); + } + + } + } + + static ClusterBlocks randomClusterBlocks() { + ClusterBlocks.Builder builder = ClusterBlocks.builder(); + int randomGlobalBlocks = randomIntBetween(0, 10); + for (int i = 0; i < randomGlobalBlocks; i++) { + builder.addGlobalBlock(randomClusterBlock()); + } + + int randomIndices = randomIntBetween(0, 10); + for (int i = 0; i < randomIndices; i++) { + int randomIndexBlocks = randomIntBetween(0, 10); + for (int j = 0; j < randomIndexBlocks; j++) { + builder.addIndexBlock("index-" + i, randomClusterBlock()); + } + } + return builder.build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java new file mode 100644 index 0000000000000..1f7a5e8bfffb1 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterStateCustomsTests.java @@ -0,0 +1,260 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterState.Custom; +import org.opensearch.cluster.SnapshotsInProgress; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.compress.DeflateCompressor; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.io.stream.NamedWriteableRegistry; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.index.translog.transfer.BlobStoreTransferService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; +import org.opensearch.snapshots.Snapshot; +import org.opensearch.snapshots.SnapshotId; +import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.threadpool.TestThreadPool; +import org.opensearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static org.opensearch.Version.CURRENT; +import static org.opensearch.cluster.SnapshotsInProgress.State.INIT; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CUSTOM_DELIMITER; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteClusterStateCustoms.CLUSTER_STATE_CUSTOM; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RemoteClusterStateCustomsTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final String CUSTOM_TYPE = "test-custom"; + private static final long STATE_VERSION = 3L; + private String clusterUUID; + private BlobStoreTransferService blobStoreTransferService; + private BlobStoreRepository blobStoreRepository; + private String clusterName; + private ClusterSettings clusterSettings; + private Compressor compressor; + private NamedWriteableRegistry namedWriteableRegistry; + private final ThreadPool threadPool = new TestThreadPool(getClass().getName()); + + @Before + public void setup() { + clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + this.clusterUUID = "test-cluster-uuid"; + this.blobStoreTransferService = mock(BlobStoreTransferService.class); + this.blobStoreRepository = mock(BlobStoreRepository.class); + BlobPath blobPath = new BlobPath().add("/path"); + when(blobStoreRepository.basePath()).thenReturn(blobPath); + when(blobStoreRepository.getCompressor()).thenReturn(new DeflateCompressor()); + compressor = new NoneCompressor(); + namedWriteableRegistry = writableRegistry(); + this.clusterName = "test-cluster-name"; + } + + @After + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdown(); + } + + public void testClusterUUID() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForUpload.clusterUUID(), is(clusterUUID)); + + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + TEST_BLOB_NAME, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.clusterUUID(), is(clusterUUID)); + } + + public void testFullBlobName() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForUpload.getFullBlobName(), nullValue()); + + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + TEST_BLOB_NAME, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.getFullBlobName(), is(TEST_BLOB_NAME)); + } + + public void testBlobFileName() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForUpload.getBlobFileName(), nullValue()); + + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + TEST_BLOB_NAME, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.getBlobFileName(), is(TEST_BLOB_FILE_NAME)); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/clusterStateCustoms"; + RemoteClusterStateCustoms remoteObjectForDownload = new RemoteClusterStateCustoms( + uploadedFile, + "test-custom", + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThat(remoteObjectForDownload.getBlobPathTokens(), is(new String[] { "user", "local", "opensearch", "clusterStateCustoms" })); + } + + public void testBlobPathParameters() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertThat(params.getPathTokens(), is(List.of(RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN))); + String expectedPrefix = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, "test-custom"); + assertThat(params.getFilePrefix(), is(expectedPrefix)); + } + + public void testGenerateBlobFileName() { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + String expectedPrefix = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, "test-custom"); + assertThat(nameTokens[0], is(expectedPrefix)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[1]), is(STATE_VERSION)); + assertThat(RemoteStoreUtils.invertLong(nameTokens[2]), lessThanOrEqualTo(System.currentTimeMillis())); + assertThat(nameTokens[3], is(String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION))); + + } + + public void testGetUploadedMetadata() throws IOException { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + "test-custom", + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + String expectedPrefix = String.join(CUSTOM_DELIMITER, CLUSTER_STATE_CUSTOM, "test-custom"); + assertThat(uploadedMetadata.getComponent(), is(expectedPrefix)); + assertThat(uploadedMetadata.getUploadedFilename(), is(remoteObjectForUpload.getFullBlobName())); + } + } + + public void testSerDe() throws IOException { + Custom clusterStateCustoms = getClusterStateCustom(); + RemoteClusterStateCustoms remoteObjectForUpload = new RemoteClusterStateCustoms( + clusterStateCustoms, + SnapshotsInProgress.TYPE, + STATE_VERSION, + clusterUUID, + compressor, + namedWriteableRegistry + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertThat(inputStream.available(), greaterThan(0)); + Custom readclusterStateCustoms = remoteObjectForUpload.deserialize(inputStream); + assertThat(readclusterStateCustoms, is(clusterStateCustoms)); + } + } + + private Custom getClusterStateCustom() { + return SnapshotsInProgress.of( + List.of( + new SnapshotsInProgress.Entry( + new Snapshot("repo", new SnapshotId("test-snapshot", "test-snapshot-uuid")), + false, + false, + INIT, + emptyList(), + emptyList(), + 0L, + 0L, + emptyMap(), + emptyMap(), + CURRENT, + false + ) + ) + ); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java new file mode 100644 index 0000000000000..0c46960938798 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteDiscoveryNodesTests.java @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.Version; +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.node.DiscoveryNodeRole; +import org.opensearch.cluster.node.DiscoveryNodes; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.CLUSTER_STATE_EPHEMERAL_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteDiscoveryNodes.DISCOVERY_NODES; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RemoteDiscoveryNodesTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(TEST_BLOB_NAME, clusterUUID, compressor); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(TEST_BLOB_NAME, clusterUUID, compressor); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteClusterBlocks remoteObjectForDownload = new RemoteClusterBlocks(TEST_BLOB_NAME, clusterUUID, compressor); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/discovery-nodes"; + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(uploadedFile, clusterUUID, compressor); + assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "discovery-nodes" }); + } + + public void testBlobPathParameters() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(CLUSTER_STATE_EPHEMERAL_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), DISCOVERY_NODES); + } + + public void testGenerateBlobFileName() { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], DISCOVERY_NODES); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION)); + } + + public void testGetUploadedMetadata() throws IOException { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), DISCOVERY_NODES); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + + public void testSerDe() throws IOException { + DiscoveryNodes nodes = getDiscoveryNodes(); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + DiscoveryNodes readDiscoveryNodes = remoteObjectForUpload.deserialize(inputStream); + assertEquals(nodes.getSize(), readDiscoveryNodes.getSize()); + nodes.getNodes().forEach((nodeId, node) -> assertEquals(readDiscoveryNodes.get(nodeId), node)); + assertEquals(nodes.getClusterManagerNodeId(), readDiscoveryNodes.getClusterManagerNodeId()); + } + } + + public void testExceptionDuringSerialization() throws IOException { + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + RemoteDiscoveryNodes remoteObjectForUpload = new RemoteDiscoveryNodes(nodes, METADATA_VERSION, clusterUUID, compressor); + doThrow(new IOException("mock-exception")).when(nodes).writeTo(any()); + IOException iea = assertThrows(IOException.class, remoteObjectForUpload::serialize); + assertEquals("Failed to serialize remote discovery nodes", iea.getMessage()); + } + + public void testExceptionDuringDeserialize() throws IOException { + DiscoveryNodes nodes = mock(DiscoveryNodes.class); + InputStream in = mock(InputStream.class); + when(in.read(any(byte[].class))).thenThrow(new IOException("mock-exception")); + String uploadedFile = "user/local/opensearch/discovery-nodes"; + RemoteDiscoveryNodes remoteObjectForDownload = new RemoteDiscoveryNodes(uploadedFile, clusterUUID, compressor); + IOException ioe = assertThrows(IOException.class, () -> remoteObjectForDownload.deserialize(in)); + assertEquals("Failed to deserialize remote discovery nodes", ioe.getMessage()); + } + + private DiscoveryNodes getDiscoveryNodes() { + return DiscoveryNodes.builder() + .add( + new DiscoveryNode( + "name_" + 1, + "node_" + 1, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), + Version.CURRENT + ) + ) + .add( + new DiscoveryNode( + "name_" + 2, + "node_" + 2, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), + Version.CURRENT + ) + ) + .add( + new DiscoveryNode( + "name_" + 3, + "node_" + 3, + buildNewFakeTransportAddress(), + Collections.emptyMap(), + new HashSet<>(randomSubsetOf(DiscoveryNodeRole.BUILT_IN_ROLES)), + Version.CURRENT + ) + ) + .localNodeId("name_1") + .clusterManagerNodeId("name_2") + .build(); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java new file mode 100644 index 0000000000000..d883eabf9fbc9 --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteHashesOfConsistentSettingsTests.java @@ -0,0 +1,193 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterModule; +import org.opensearch.cluster.metadata.DiffableStringMap; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateAttributesManager.CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN; +import static org.opensearch.gateway.remote.model.RemoteHashesOfConsistentSettings.HASHES_OF_CONSISTENT_SETTINGS; + +public class RemoteHashesOfConsistentSettingsTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor + ); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + TEST_BLOB_NAME, + clusterUUID, + compressor + ); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor + ); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + TEST_BLOB_NAME, + clusterUUID, + compressor + ); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor + ); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + TEST_BLOB_NAME, + clusterUUID, + compressor + ); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/hashes-of-consistent-settings"; + RemoteHashesOfConsistentSettings remoteObjectForDownload = new RemoteHashesOfConsistentSettings( + uploadedFile, + clusterUUID, + compressor + ); + assertArrayEquals( + remoteObjectForDownload.getBlobPathTokens(), + new String[] { "user", "local", "opensearch", "hashes-of-consistent-settings" } + ); + } + + public void testBlobPathParameters() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(GLOBAL_METADATA_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), HASHES_OF_CONSISTENT_SETTINGS); + } + + public void testGenerateBlobFileName() { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], HASHES_OF_CONSISTENT_SETTINGS); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(CLUSTER_STATE_ATTRIBUTES_CURRENT_CODEC_VERSION)); + } + + public void testGetUploadedMetadata() throws IOException { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), HASHES_OF_CONSISTENT_SETTINGS); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + + public void testSerDe() throws IOException { + DiffableStringMap hashesOfConsistentSettings = getHashesOfConsistentSettings(); + RemoteHashesOfConsistentSettings remoteObjectForUpload = new RemoteHashesOfConsistentSettings( + hashesOfConsistentSettings, + METADATA_VERSION, + clusterUUID, + compressor + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + DiffableStringMap readHashesOfConsistentSettings = remoteObjectForUpload.deserialize(inputStream); + assertEquals(hashesOfConsistentSettings.entrySet(), readHashesOfConsistentSettings.entrySet()); + } + } + + private DiffableStringMap getHashesOfConsistentSettings() { + Map hashesOfConsistentSettings = new HashMap<>(); + hashesOfConsistentSettings.put("secure-setting-key", "secure-setting-value"); + return new DiffableStringMap(hashesOfConsistentSettings); + } +} diff --git a/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadataTests.java b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadataTests.java new file mode 100644 index 0000000000000..4061ab4a6b5ef --- /dev/null +++ b/server/src/test/java/org/opensearch/gateway/remote/model/RemoteTransientSettingsMetadataTests.java @@ -0,0 +1,200 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.gateway.remote.model; + +import org.opensearch.cluster.ClusterModule; +import org.opensearch.common.blobstore.BlobPath; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.remote.BlobPathParameters; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.compress.Compressor; +import org.opensearch.core.compress.NoneCompressor; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.gateway.remote.ClusterMetadataManifest; +import org.opensearch.gateway.remote.RemoteClusterStateUtils; +import org.opensearch.index.remote.RemoteStoreUtils; +import org.opensearch.indices.IndicesModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; +import static org.opensearch.gateway.remote.RemoteClusterStateUtils.GLOBAL_METADATA_CURRENT_CODEC_VERSION; +import static org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata.TRANSIENT_SETTING_METADATA; + +public class RemoteTransientSettingsMetadataTests extends OpenSearchTestCase { + private static final String TEST_BLOB_NAME = "/test-path/test-blob-name"; + private static final String TEST_BLOB_PATH = "test-path"; + private static final String TEST_BLOB_FILE_NAME = "test-blob-name"; + private static final long METADATA_VERSION = 3L; + private String clusterUUID; + private Compressor compressor; + private NamedXContentRegistry namedXContentRegistry; + + @Before + public void setup() { + this.clusterUUID = "test-cluster-uuid"; + compressor = new NoneCompressor(); + namedXContentRegistry = new NamedXContentRegistry( + Stream.of( + NetworkModule.getNamedXContents().stream(), + IndicesModule.getNamedXContents().stream(), + ClusterModule.getNamedXWriteables().stream() + ).flatMap(Function.identity()).collect(toList()) + ); + } + + public void testClusterUUID() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForUpload.clusterUUID(), clusterUUID); + + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.clusterUUID(), clusterUUID); + } + + public void testFullBlobName() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getFullBlobName()); + + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getFullBlobName(), TEST_BLOB_NAME); + } + + public void testBlobFileName() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertNull(remoteObjectForUpload.getBlobFileName()); + + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + TEST_BLOB_NAME, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertEquals(remoteObjectForDownload.getBlobFileName(), TEST_BLOB_FILE_NAME); + } + + public void testBlobPathTokens() { + String uploadedFile = "user/local/opensearch/settings"; + RemoteTransientSettingsMetadata remoteObjectForDownload = new RemoteTransientSettingsMetadata( + uploadedFile, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertArrayEquals(remoteObjectForDownload.getBlobPathTokens(), new String[] { "user", "local", "opensearch", "settings" }); + } + + public void testBlobPathParameters() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + BlobPathParameters params = remoteObjectForUpload.getBlobPathParameters(); + assertEquals(params.getPathTokens(), List.of(RemoteClusterStateUtils.GLOBAL_METADATA_PATH_TOKEN)); + assertEquals(params.getFilePrefix(), TRANSIENT_SETTING_METADATA); + } + + public void testGenerateBlobFileName() { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + String blobFileName = remoteObjectForUpload.generateBlobFileName(); + String[] nameTokens = blobFileName.split(RemoteClusterStateUtils.DELIMITER); + assertEquals(nameTokens[0], TRANSIENT_SETTING_METADATA); + assertEquals(RemoteStoreUtils.invertLong(nameTokens[1]), METADATA_VERSION); + assertTrue(RemoteStoreUtils.invertLong(nameTokens[2]) <= System.currentTimeMillis()); + assertEquals(nameTokens[3], String.valueOf(GLOBAL_METADATA_CURRENT_CODEC_VERSION)); + + } + + public void testGetUploadedMetadata() throws IOException { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + assertThrows(AssertionError.class, remoteObjectForUpload::getUploadedMetadata); + + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(new BlobPath().add(TEST_BLOB_PATH)); + ClusterMetadataManifest.UploadedMetadata uploadedMetadata = remoteObjectForUpload.getUploadedMetadata(); + assertEquals(uploadedMetadata.getComponent(), TRANSIENT_SETTING_METADATA); + assertEquals(uploadedMetadata.getUploadedFilename(), remoteObjectForUpload.getFullBlobName()); + } + } + + public void testSerDe() throws IOException { + Settings settings = getSettings(); + RemoteTransientSettingsMetadata remoteObjectForUpload = new RemoteTransientSettingsMetadata( + settings, + METADATA_VERSION, + clusterUUID, + compressor, + namedXContentRegistry + ); + try (InputStream inputStream = remoteObjectForUpload.serialize()) { + remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath()); + assertTrue(inputStream.available() > 0); + Settings readsettings = remoteObjectForUpload.deserialize(inputStream); + assertEquals(readsettings, settings); + } + } + + private Settings getSettings() { + return Settings.builder().put("random_index_setting_" + randomAlphaOfLength(3), randomAlphaOfLength(5)).build(); + } +}