forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Create SystemIndexRegistry with helper method matchesSystemIndex (ope…
…nsearch-project#14415) * Create new extension point in SystemIndexPlugin for a single plugin to get registered system indices Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add to CHANGELOG Signed-off-by: Craig Perkins <cwperx@amazon.com> * WIP on system indices from IndexNameExpressionResolver Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add test in IndexNameExpressionResolverTests Signed-off-by: Craig Perkins <cwperx@amazon.com> * Remove changes in SystemIndexPlugin Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add method in IndexNameExpressionResolver to get matching system indices Signed-off-by: Craig Perkins <cwperx@amazon.com> * Show how resolver can be chained to get system indices Signed-off-by: Craig Perkins <cwperx@amazon.com> * Fix forbiddenApis check Signed-off-by: Craig Perkins <cwperx@amazon.com> * Update CHANGELOG Signed-off-by: Craig Perkins <cwperx@amazon.com> * Make SystemIndices internal Signed-off-by: Craig Perkins <cwperx@amazon.com> * Remove unneeded changes Signed-off-by: Craig Perkins <cwperx@amazon.com> * Fix CI failures Signed-off-by: Craig Perkins <cwperx@amazon.com> * Fix precommit errors Signed-off-by: Craig Perkins <cwperx@amazon.com> * Use Regex instead of WildcardMatcher Signed-off-by: Craig Perkins <cwperx@amazon.com> * Address code review feedback Signed-off-by: Craig Perkins <cwperx@amazon.com> * Allow caller to pass index expressions Signed-off-by: Craig Perkins <cwperx@amazon.com> * Create SystemIndexRegistry Signed-off-by: Craig Perkins <cwperx@amazon.com> * Update CHANGELOG Signed-off-by: Craig Perkins <cwperx@amazon.com> * Remove singleton limitation Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add javadoc Signed-off-by: Craig Perkins <cwperx@amazon.com> * Add @experimentalapi annotation Signed-off-by: Craig Perkins <cwperx@amazon.com> --------- Signed-off-by: Craig Perkins <cwperx@amazon.com> Signed-off-by: Kaushal Kumar <ravi.kaushal97@gmail.com>
- Loading branch information
1 parent
8f5ef47
commit b6d4c40
Showing
6 changed files
with
242 additions
and
97 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
130 changes: 130 additions & 0 deletions
130
server/src/main/java/org/opensearch/indices/SystemIndexRegistry.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.indices; | ||
|
|
||
| import org.apache.lucene.util.automaton.Automaton; | ||
| import org.apache.lucene.util.automaton.Operations; | ||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
| import org.opensearch.common.collect.Tuple; | ||
| import org.opensearch.common.regex.Regex; | ||
| import org.opensearch.tasks.TaskResultsService; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Collection; | ||
| import java.util.Collections; | ||
| import java.util.Comparator; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import static java.util.Collections.singletonList; | ||
| import static java.util.Collections.singletonMap; | ||
| import static java.util.Collections.unmodifiableMap; | ||
| import static org.opensearch.tasks.TaskResultsService.TASK_INDEX; | ||
|
|
||
| /** | ||
| * This class holds the {@link SystemIndexDescriptor} objects that represent system indices the | ||
| * node knows about. This class also contains static methods that identify if index expressions match | ||
| * registered system index patterns | ||
| * | ||
| * @opensearch.api | ||
| */ | ||
| @ExperimentalApi | ||
| public class SystemIndexRegistry { | ||
| private static final SystemIndexDescriptor TASK_INDEX_DESCRIPTOR = new SystemIndexDescriptor(TASK_INDEX + "*", "Task Result Index"); | ||
| private static final Map<String, Collection<SystemIndexDescriptor>> SERVER_SYSTEM_INDEX_DESCRIPTORS = singletonMap( | ||
| TaskResultsService.class.getName(), | ||
| singletonList(TASK_INDEX_DESCRIPTOR) | ||
| ); | ||
|
|
||
| private volatile static String[] SYSTEM_INDEX_PATTERNS = new String[0]; | ||
| volatile static Collection<SystemIndexDescriptor> SYSTEM_INDEX_DESCRIPTORS = Collections.emptyList(); | ||
|
|
||
| static void register(Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesDescriptors) { | ||
| final Map<String, Collection<SystemIndexDescriptor>> descriptorsMap = buildSystemIndexDescriptorMap(pluginAndModulesDescriptors); | ||
| checkForOverlappingPatterns(descriptorsMap); | ||
| List<SystemIndexDescriptor> descriptors = pluginAndModulesDescriptors.values() | ||
| .stream() | ||
| .flatMap(Collection::stream) | ||
| .collect(Collectors.toList()); | ||
| descriptors.add(TASK_INDEX_DESCRIPTOR); | ||
|
|
||
| SYSTEM_INDEX_DESCRIPTORS = descriptors.stream().collect(Collectors.toUnmodifiableList()); | ||
| SYSTEM_INDEX_PATTERNS = descriptors.stream().map(SystemIndexDescriptor::getIndexPattern).toArray(String[]::new); | ||
| } | ||
|
|
||
| public static List<String> matchesSystemIndexPattern(String... indexExpressions) { | ||
| return Arrays.stream(indexExpressions) | ||
| .filter(pattern -> Regex.simpleMatch(SYSTEM_INDEX_PATTERNS, pattern)) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| /** | ||
| * Given a collection of {@link SystemIndexDescriptor}s and their sources, checks to see if the index patterns of the listed | ||
| * descriptors overlap with any of the other patterns. If any do, throws an exception. | ||
| * | ||
| * @param sourceToDescriptors A map of source (plugin) names to the SystemIndexDescriptors they provide. | ||
| * @throws IllegalStateException Thrown if any of the index patterns overlaps with another. | ||
| */ | ||
| static void checkForOverlappingPatterns(Map<String, Collection<SystemIndexDescriptor>> sourceToDescriptors) { | ||
| List<Tuple<String, SystemIndexDescriptor>> sourceDescriptorPair = sourceToDescriptors.entrySet() | ||
| .stream() | ||
| .flatMap(entry -> entry.getValue().stream().map(descriptor -> new Tuple<>(entry.getKey(), descriptor))) | ||
| .sorted(Comparator.comparing(d -> d.v1() + ":" + d.v2().getIndexPattern())) // Consistent ordering -> consistent error message | ||
| .collect(Collectors.toList()); | ||
|
|
||
| // This is O(n^2) with the number of system index descriptors, and each check is quadratic with the number of states in the | ||
| // automaton, but the absolute number of system index descriptors should be quite small (~10s at most), and the number of states | ||
| // per pattern should be low as well. If these assumptions change, this might need to be reworked. | ||
| sourceDescriptorPair.forEach(descriptorToCheck -> { | ||
| List<Tuple<String, SystemIndexDescriptor>> descriptorsMatchingThisPattern = sourceDescriptorPair.stream() | ||
|
|
||
| .filter(d -> descriptorToCheck.v2() != d.v2()) // Exclude the pattern currently being checked | ||
| .filter(d -> overlaps(descriptorToCheck.v2(), d.v2())) | ||
| .collect(Collectors.toList()); | ||
| if (descriptorsMatchingThisPattern.isEmpty() == false) { | ||
| throw new IllegalStateException( | ||
| "a system index descriptor [" | ||
| + descriptorToCheck.v2() | ||
| + "] from [" | ||
| + descriptorToCheck.v1() | ||
| + "] overlaps with other system index descriptors: [" | ||
| + descriptorsMatchingThisPattern.stream() | ||
| .map(descriptor -> descriptor.v2() + " from [" + descriptor.v1() + "]") | ||
| .collect(Collectors.joining(", ")) | ||
| ); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| private static boolean overlaps(SystemIndexDescriptor a1, SystemIndexDescriptor a2) { | ||
| Automaton a1Automaton = Regex.simpleMatchToAutomaton(a1.getIndexPattern()); | ||
| Automaton a2Automaton = Regex.simpleMatchToAutomaton(a2.getIndexPattern()); | ||
| return Operations.isEmpty(Operations.intersection(a1Automaton, a2Automaton)) == false; | ||
| } | ||
|
|
||
| private static Map<String, Collection<SystemIndexDescriptor>> buildSystemIndexDescriptorMap( | ||
| Map<String, Collection<SystemIndexDescriptor>> pluginAndModulesMap | ||
| ) { | ||
| final Map<String, Collection<SystemIndexDescriptor>> map = new HashMap<>( | ||
| pluginAndModulesMap.size() + SERVER_SYSTEM_INDEX_DESCRIPTORS.size() | ||
| ); | ||
| map.putAll(pluginAndModulesMap); | ||
| // put the server items last since we expect less of them | ||
| SERVER_SYSTEM_INDEX_DESCRIPTORS.forEach((source, descriptors) -> { | ||
| if (map.putIfAbsent(source, descriptors) != null) { | ||
| throw new IllegalArgumentException( | ||
| "plugin or module attempted to define the same source [" + source + "] as a built-in system index" | ||
| ); | ||
| } | ||
| }); | ||
| return unmodifiableMap(map); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.