Skip to content

Commit

Permalink
Add integ tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Ashish Singh <ssashish@amazon.com>
  • Loading branch information
@ashking94
ashking94 committed Mar 20, 2024
1 parent 08cb85c commit 45aaf5b
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 1 deletion.
84 changes: 83 additions & 1 deletion server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.opensearch.action.support.PlainActionFuture;
import org.opensearch.client.Client;
import org.opensearch.client.Requests;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.io.PathUtils;
import org.opensearch.common.settings.Settings;
Expand All @@ -26,6 +27,7 @@
import org.opensearch.core.rest.RestStatus;
import org.opensearch.index.IndexService;
import org.opensearch.index.IndexSettings;
import org.opensearch.index.remote.RemoteStorePathType;
import org.opensearch.index.shard.IndexShard;
import org.opensearch.indices.IndicesService;
import org.opensearch.indices.replication.common.ReplicationType;
Expand All @@ -43,14 +45,15 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
Expand Down Expand Up @@ -257,6 +260,85 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2);
}

/**
* In this test, we validate presence of remote_store custom data in index metadata for standard index creation and
* on snapshot restore.
*/
public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
internalCluster().startDataOnlyNode();
String indexName1 = "testindex1";
String indexName2 = "testindex2";
String snapshotRepoName = "test-restore-snapshot-repo";
String snapshotName1 = "test-restore-snapshot1";
Path absolutePath1 = randomRepoPath().toAbsolutePath();
logger.info("Snapshot Path [{}]", absolutePath1);
String restoredIndexName1version1 = indexName1 + "-restored-1";
String restoredIndexName1version2 = indexName1 + "-restored-2";

createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
Client client = client();
Settings indexSettings = getIndexSettings(1, 0).build();
createIndex(indexName1, indexSettings);

indexDocuments(client, indexName1, randomIntBetween(5, 10));
ensureGreen(indexName1);
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);

logger.info("--> snapshot");
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
assertTrue(snapshotInfo.successfulShards() > 0);
assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());

RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
.setWaitForCompletion(false)
.setRenamePattern(indexName1)
.setRenameReplacement(restoredIndexName1version1)
.get();
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
ensureGreen(restoredIndexName1version1);
validateRemoteStorePathType(restoredIndexName1version1, RemoteStorePathType.FIXED);

client(clusterManagerNode).admin()
.cluster()
.prepareUpdateSettings()
.setTransientSettings(
Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), RemoteStorePathType.HASHED_PREFIX)
)
.get();

restoreSnapshotResponse = client.admin()
.cluster()
.prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
.setWaitForCompletion(false)
.setRenamePattern(indexName1)
.setRenameReplacement(restoredIndexName1version2)
.get();
assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
ensureGreen(restoredIndexName1version2);
validateRemoteStorePathType(restoredIndexName1version2, RemoteStorePathType.HASHED_PREFIX);

// Create index with cluster setting cluster.remote_store.index.path.prefix.type as hashed_prefix.
indexSettings = getIndexSettings(1, 0).build();
createIndex(indexName2, indexSettings);
ensureGreen(indexName2);
validateRemoteStorePathType(indexName2, RemoteStorePathType.HASHED_PREFIX);

// Validating that custom data has not changed for indexes which were created before the cluster setting got updated
validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
}

private void validateRemoteStorePathType(String index, RemoteStorePathType pathType) {
ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
// Validate that the remote_store custom data is present in index metadata for the created index.
Map<String, String> remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
assertNotNull(remoteCustomData);
assertEquals(pathType.toString(), remoteCustomData.get(RemoteStorePathType.NAME));
}

public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
String primary = internalCluster().startDataOnlyNode();
Expand Down
57 changes: 57 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.opensearch.action.index.IndexRequestBuilder;
import org.opensearch.client.Client;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.block.ClusterBlocks;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.MappingMetadata;
Expand Down Expand Up @@ -151,6 +152,62 @@ public void testParallelRestoreOperations() {
assertThat(client.prepareGet(restoredIndexName2, docId2).get().isExists(), equalTo(true));
}

/**
* In this test, we test that an index created does not have any remote_store custom data in index metadata at the
* time of index creation and after snapshot restore.
*/
public void testNoRemoteStoreCustomDataOnIndexCreationAndRestore() {
String indexName1 = "testindex1";
String repoName = "test-restore-snapshot-repo";
String snapshotName1 = "test-restore-snapshot1";
Path absolutePath = randomRepoPath().toAbsolutePath();
logger.info("Path [{}]", absolutePath);
String restoredIndexName1 = indexName1 + "-restored";
String expectedValue = "expected";

Client client = client();
// Write a document
String docId = Integer.toString(randomInt());
index(indexName1, "_doc", docId, "value", expectedValue);

createRepository(repoName, "fs", absolutePath);

logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client.admin()
.cluster()
.prepareCreateSnapshot(repoName, snapshotName1)
.setWaitForCompletion(true)
.setIndices(indexName1)
.get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(
createSnapshotResponse.getSnapshotInfo().successfulShards(),
equalTo(createSnapshotResponse.getSnapshotInfo().totalShards())
);
assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));

ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();

// Validate that the remote_store custom data is not present in index metadata for the created index.
assertNull(state.metadata().index(indexName1).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));

RestoreSnapshotResponse restoreSnapshotResponse1 = client.admin()
.cluster()
.prepareRestoreSnapshot(repoName, snapshotName1)
.setWaitForCompletion(false)
.setRenamePattern(indexName1)
.setRenameReplacement(restoredIndexName1)
.get();
assertThat(restoreSnapshotResponse1.status(), equalTo(RestStatus.ACCEPTED));
ensureGreen(restoredIndexName1);
assertThat(client.prepareGet(restoredIndexName1, docId).get().isExists(), equalTo(true));

state = client().admin().cluster().prepareState().execute().actionGet().getState();

// Validate that the remote_store custom data is not present in index metadata for the restored index.
assertNull(state.metadata().index(restoredIndexName1).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
}

public void testParallelRestoreOperationsFromSingleSnapshot() throws Exception {
String indexName1 = "testindex1";
String indexName2 = "testindex2";
Expand Down

0 comments on commit 45aaf5b

Please sign in to comment.