Skip to content

Commit

Permalink
Support index.auto_expand_replicas 0-all for .plugins-ml-config (open…
Browse files Browse the repository at this point in the history 
…search-project#3017)

Signed-off-by: b4sjoo <sicheng.song@outlook.com>
  • Loading branch information
@b4sjoo
b4sjoo authored Oct 1, 2024
1 parent 9de2d23 commit 37ab6fa
Show file tree
Hide file tree
Showing 6 changed files with 56 additions and 17 deletions.
2 changes: 1 addition & 1 deletion common/src/main/java/org/opensearch/ml/common/CommonValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public class CommonValue {
public static final Integer ML_TASK_INDEX_SCHEMA_VERSION = 3;
public static final Integer ML_CONNECTOR_SCHEMA_VERSION = 3;
public static final String ML_CONFIG_INDEX = ".plugins-ml-config";
public static final Integer ML_CONFIG_INDEX_SCHEMA_VERSION = 3;
public static final Integer ML_CONFIG_INDEX_SCHEMA_VERSION = 4;
public static final String ML_CONTROLLER_INDEX = ".plugins-ml-controller";
public static final Integer ML_CONTROLLER_INDEX_SCHEMA_VERSION = 1;
public static final String ML_MAP_RESPONSE_KEY = "response";
Expand Down
23 changes: 19 additions & 4 deletions common/src/main/java/org/opensearch/ml/common/utils/IndexUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,24 @@
@Log4j2
public class IndexUtils {

// This setting is for index creation.
public static final Map<String, Object> INDEX_SETTINGS = Map.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-1");
/**
* Default settings for index creation with a single shard and one replica.
* - Sets the number of shards to 1 for better performance in small indices.
* - Uses auto-expand replicas (0-1) to ensure high availability while minimizing resource usage.
*/
public static final Map<String, Object> DEFAULT_INDEX_SETTINGS = Map
.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-1");
/**
* Default settings for index creation with replicas on all nodes.
* - Sets the number of shards to 1 for better performance in small indices.
* - Uses auto-expand replicas (0-all) to ensure a replica on every node, maximizing availability.
* - Caution: This can significantly increase storage requirements and indexing load.
* - Suitable for small, critical indices where maximum redundancy is required.
*/
public static final Map<String, Object> ALL_NODES_REPLICA_INDEX_SETTINGS = Map
.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-all");

// This setting is for index update only, so only dynamic settings should be contained!
public static final Map<String, Object> UPDATED_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-1");
// Note: This does not include static settings like number of shards, which can't be changed after index creation.
public static final Map<String, Object> UPDATED_DEFAULT_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-1");
public static final Map<String, Object> UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-all");
}
22 changes: 18 additions & 4 deletions common/src/test/java/org/opensearch/ml/common/utils/IndexUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,32 @@
public class IndexUtilsTest {

@Test
public void testIndexSettingsContainsExpectedValues() {
Map<String, Object> indexSettings = IndexUtils.INDEX_SETTINGS;
public void testDefaultIndexSettingsContainsExpectedValues() {
Map<String, Object> indexSettings = IndexUtils.DEFAULT_INDEX_SETTINGS;
assertEquals("index.number_of_shards should be 1", indexSettings.get("index.number_of_shards"), "1");
assertEquals("index.auto_expand_replicas should be 0-1", indexSettings.get("index.auto_expand_replicas"), "0-1");
assertEquals("INDEX_SETTINGS should contain exactly 2 settings", 2, indexSettings.size());
}

@Test
public void testUpdatedIndexSettingsContainsExpectedValues() {
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_INDEX_SETTINGS;
public void testAllNodesReplicaIndexSettingsContainsExpectedValues() {
Map<String, Object> indexSettings = IndexUtils.ALL_NODES_REPLICA_INDEX_SETTINGS;
assertEquals("index.number_of_shards should be 1", indexSettings.get("index.number_of_shards"), "1");
assertEquals("index.auto_expand_replicas should be 0-all", indexSettings.get("index.auto_expand_replicas"), "0-all");
assertEquals("INDEX_SETTINGS should contain exactly 2 settings", 2, indexSettings.size());
}

@Test
public void testUpdatedDefaultIndexSettingsContainsExpectedValues() {
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_DEFAULT_INDEX_SETTINGS;
assertEquals("index.auto_expand_replicas should be 0-1", updatedIndexSettings.get("index.auto_expand_replicas"), "0-1");
assertEquals("INDEX_SETTINGS should contain exactly 1 settings", 1, updatedIndexSettings.size());
}

@Test
public void testUpdatedAllNodesReplicaIndexSettingsContainsExpectedValues() {
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS;
assertEquals("index.auto_expand_replicas should be 0-all", updatedIndexSettings.get("index.auto_expand_replicas"), "0-all");
assertEquals("INDEX_SETTINGS should contain exactly 1 settings", 1, updatedIndexSettings.size());
}
}
4 changes: 2 additions & 2 deletions memory/src/main/java/org/opensearch/ml/memory/index/ConversationMetaIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.opensearch.ml.memory.index;

import static org.opensearch.ml.common.conversation.ConversationalIndexConstants.META_INDEX_NAME;
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;

import java.io.IOException;
import java.time.Instant;
Expand Down Expand Up @@ -90,7 +90,7 @@ public void initConversationMetaIndexIfAbsent(ActionListener<Boolean> listener)
CreateIndexRequest request = Requests
.createIndexRequest(META_INDEX_NAME)
.mapping(ConversationalIndexConstants.META_MAPPING, XContentType.JSON)
.settings(INDEX_SETTINGS);
.settings(DEFAULT_INDEX_SETTINGS);
try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) {
ActionListener<Boolean> internalListener = ActionListener.runBefore(listener, () -> threadContext.restore());
ActionListener<CreateIndexResponse> al = ActionListener.wrap(createIndexResponse -> {
Expand Down
4 changes: 2 additions & 2 deletions memory/src/main/java/org/opensearch/ml/memory/index/InteractionsIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.opensearch.ml.memory.index;

import static org.opensearch.ml.common.conversation.ConversationalIndexConstants.INTERACTIONS_INDEX_NAME;
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;

import java.io.IOException;
import java.time.Instant;
Expand Down Expand Up @@ -93,7 +93,7 @@ public void initInteractionsIndexIfAbsent(ActionListener<Boolean> listener) {
CreateIndexRequest request = Requests
.createIndexRequest(INTERACTIONS_INDEX_NAME)
.mapping(ConversationalIndexConstants.INTERACTIONS_MAPPINGS, XContentType.JSON)
.settings(INDEX_SETTINGS);
.settings(DEFAULT_INDEX_SETTINGS);
try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) {
ActionListener<Boolean> internalListener = ActionListener.runBefore(listener, () -> threadContext.restore());
ActionListener<CreateIndexResponse> al = ActionListener.wrap(r -> {
Expand Down
18 changes: 14 additions & 4 deletions ml-algorithms/src/main/java/org/opensearch/ml/engine/indices/MLIndicesHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@

import static org.opensearch.ml.common.CommonValue.META;
import static org.opensearch.ml.common.CommonValue.SCHEMA_VERSION_FIELD;
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.ALL_NODES_REPLICA_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS;
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_DEFAULT_INDEX_SETTINGS;

import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -108,7 +110,9 @@ public void initMLIndexIfAbsent(MLIndex index, ActionListener<Boolean> listener)
internalListener.onFailure(e);
}
});
CreateIndexRequest request = new CreateIndexRequest(indexName).mapping(mapping, XContentType.JSON).settings(INDEX_SETTINGS);
CreateIndexRequest request = new CreateIndexRequest(indexName)
.mapping(mapping, XContentType.JSON)
.settings(indexName.equals(MLIndex.CONFIG.getIndexName()) ? ALL_NODES_REPLICA_INDEX_SETTINGS : DEFAULT_INDEX_SETTINGS);
client.admin().indices().create(request, actionListener);
} else {
log.debug("index:{} is already created", indexName);
Expand All @@ -124,7 +128,13 @@ public void initMLIndexIfAbsent(MLIndex index, ActionListener<Boolean> listener)
ActionListener.wrap(response -> {
if (response.isAcknowledged()) {
UpdateSettingsRequest updateSettingRequest = new UpdateSettingsRequest();
updateSettingRequest.indices(indexName).settings(UPDATED_INDEX_SETTINGS);
updateSettingRequest
.indices(indexName)
.settings(
indexName.equals(MLIndex.CONFIG.getIndexName())
? UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS
: UPDATED_DEFAULT_INDEX_SETTINGS
);
client
.admin()
.indices()
Expand Down

0 comments on commit 37ab6fa

Please sign in to comment.