Skip to content

Commit

Permalink
Use cluster default remote store path type during snapshot restore (#…
Browse files Browse the repository at this point in the history 
…12753)

* Use cluster default remote store path type as fallback during snapshot restore

---------

Signed-off-by: Ashish Singh <ssashish@amazon.com>
  • Loading branch information
@ashking94
ashking94 authored Mar 20, 2024
1 parent 773a939 commit 0510c5b
Show file tree
Hide file tree
Showing 4 changed files with 158 additions and 7 deletions.
84 changes: 83 additions & 1 deletion server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.opensearch.action.support.PlainActionFuture;
import org.opensearch.client.Client;
import org.opensearch.client.Requests;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
Expand All @@ -26,6 +27,7 @@
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.IndexService;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.remote.RemoteStorePathType;
import org.opensearch.index.shard.IndexShard;
import org.opensearch.indices.IndicesService;
import org.opensearch.indices.replication.common.ReplicationType;
Expand All @@ -43,14 +45,15 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
Expand Down Expand Up @@ -257,6 +260,85 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2);
}

/**
* In this test, we validate presence of remote_store custom data in index metadata for standard index creation and
* on snapshot restore.
*/
public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
String snapshotName1 = "test-restore-snapshot1";
Path absolutePath1 = randomRepoPath().toAbsolutePath();
logger.info("Snapshot Path [{}]", absolutePath1);
String restoredIndexName1version1 = indexName1 + "-restored-1";
String restoredIndexName1version2 = indexName1 + "-restored-2";

createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
Client client = client();
Settings indexSettings = getIndexSettings(1, 0).build();
createIndex(indexName1, indexSettings);

indexDocuments(client, indexName1, randomIntBetween(5, 10));
ensureGreen(indexName1);
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);

logger.info("--> snapshot");
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
assertTrue(snapshotInfo.successfulShards() > 0);
assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());

RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
.setWaitForCompletion(false)
.setRenamePattern(indexName1)
.setRenameReplacement(restoredIndexName1version1)
.get();
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
ensureGreen(restoredIndexName1version1);
validateRemoteStorePathType(restoredIndexName1version1, RemoteStorePathType.FIXED);

client(clusterManagerNode).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), RemoteStorePathType.HASHED_PREFIX)
)
.get();

restoreSnapshotResponse = client.admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
.setWaitForCompletion(false)
.setRenamePattern(indexName1)
.setRenameReplacement(restoredIndexName1version2)
.get();
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
ensureGreen(restoredIndexName1version2);
validateRemoteStorePathType(restoredIndexName1version2, RemoteStorePathType.HASHED_PREFIX);

// Create index with cluster setting cluster.remote_store.index.path.prefix.type as hashed_prefix.
indexSettings = getIndexSettings(1, 0).build();
createIndex(indexName2, indexSettings);
ensureGreen(indexName2);
validateRemoteStorePathType(indexName2, RemoteStorePathType.HASHED_PREFIX);

// Validating that custom data has not changed for indexes which were created before the cluster setting got updated
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
}

private void validateRemoteStorePathType(String index, RemoteStorePathType pathType) {
ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
// Validate that the remote_store custom data is present in index metadata for the created index.
Map<String, String> remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
assertNotNull(remoteCustomData);
assertEquals(pathType.toString(), remoteCustomData.get(RemoteStorePathType.NAME));
}

public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
Expand Down
57 changes: 57 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.client.Client;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.block.ClusterBlocks;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.MappingMetadata;
Expand Down Expand Up @@ -151,6 +152,62 @@ public void testParallelRestoreOperations() {
assertThat(client.prepareGet(restoredIndexName2, docId2).get().isExists(), equalTo(true));
}

/**
* In this test, we test that an index created does not have any remote_store custom data in index metadata at the
* time of index creation and after snapshot restore.
*/
public void testNoRemoteStoreCustomDataOnIndexCreationAndRestore() {
String indexName1 = "testindex1";
String repoName = "test-restore-snapshot-repo";
String snapshotName1 = "test-restore-snapshot1";
Path absolutePath = randomRepoPath().toAbsolutePath();
logger.info("Path [{}]", absolutePath);
String restoredIndexName1 = indexName1 + "-restored";
String expectedValue = "expected";

Client client = client();
// Write a document
String docId = Integer.toString(randomInt());
index(indexName1, "_doc", docId, "value", expectedValue);

createRepository(repoName, "fs", absolutePath);

logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
.prepareCreateSnapshot(repoName, snapshotName1)
.setWaitForCompletion(true)
.setIndices(indexName1)
.get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(
createSnapshotResponse.getSnapshotInfo().successfulShards(),
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())
);
assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));

ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();

// Validate that the remote_store custom data is not present in index metadata for the created index.
assertNull(state.metadata().index(indexName1).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));

RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin()
.cluster()
.prepareRestoreSnapshot(repoName, snapshotName1)
.setWaitForCompletion(false)
.setRenamePattern(indexName1)
.setRenameReplacement(restoredIndexName1)
.get();
assertThat(restoreSnapshotResponse1.status(), equalTo(RestStatus.ACCEPTED));
ensureGreen(restoredIndexName1);
assertThat(client.prepareGet(restoredIndexName1, docId).get().isExists(), equalTo(true));

state = client().admin().cluster().prepareState().execute().actionGet().getState();

// Validate that the remote_store custom data is not present in index metadata for the restored index.
assertNull(state.metadata().index(restoredIndexName1).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
}

public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception {
String indexName1 = "testindex1";
String indexName2 = "testindex2";
Expand Down
22 changes: 17 additions & 5 deletions server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -553,11 +553,7 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
tmpImdBuilder.setRoutingNumShards(routingNumShards);
tmpImdBuilder.settings(indexSettings);
tmpImdBuilder.system(isSystem);

if (remoteStorePathResolver != null) {
String pathType = remoteStorePathResolver.resolveType().toString();
tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, Map.of(RemoteStorePathType.NAME, pathType));
}
addRemoteCustomData(tmpImdBuilder);

// Set up everything, now locally create the index to see that things are ok, and apply
IndexMetadata tempMetadata = tmpImdBuilder.build();
Expand All @@ -566,6 +562,22 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
return tempMetadata;
}

public void addRemoteCustomData(IndexMetadata.Builder tmpImdBuilder) {
if (remoteStorePathResolver != null) {
// It is possible that remote custom data exists already. In such cases, we need to only update the path type
// in the remote store custom data map.
Map<String, String> existingRemoteCustomData = tmpImdBuilder.removeCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
Map<String, String> remoteCustomData = existingRemoteCustomData == null
? new HashMap<>()
: new HashMap<>(existingRemoteCustomData);
// Determine the path type for use using the remoteStorePathResolver.
String newPathType = remoteStorePathResolver.resolveType().toString();
String oldPathType = remoteCustomData.put(RemoteStorePathType.NAME, newPathType);
logger.trace(() -> new ParameterizedMessage("Added new path type {}, replaced old path type {}", newPathType, oldPathType));
tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
}
}

private ClusterState applyCreateIndexRequestWithV1Templates(
final ClusterState currentState,
final CreateIndexClusterStateUpdateRequest request,
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/snapshots/RestoreService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,6 @@ public RestoreService(

// Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
restoreSnapshotTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.RESTORE_SNAPSHOT_KEY, true);

}

/**
Expand Down Expand Up @@ -452,6 +451,7 @@ public ClusterState execute(ClusterState currentState) {
.put(snapshotIndexMetadata.getSettings())
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
);
createIndexService.addRemoteCustomData(indexMdBuilder);
shardLimitValidator.validateShardLimit(
renamedIndexName,
snapshotIndexMetadata.getSettings(),
Expand Down

0 comments on commit 0510c5b

Please sign in to comment.