Merge branch '2.x' into backport/backport-14454-to-2.x

Signed-off-by: Pranshu Shukla <55992439+Pranshu-S@users.noreply.github.com>
opensearch-project · Sep 2, 2024 · 092397a · 092397a
2 parents 00d4d7e + 5653ed6
commit 092397a
Show file tree

Hide file tree

Showing 136 changed files with 8,650 additions and 853 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897))
 - Support filtering on a large list encoded by bitmap ([#14774](https://github.com/opensearch-project/OpenSearch/pull/14774))
 - Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153))
+- Make balanced shards allocator timebound ([#15239](https://github.com/opensearch-project/OpenSearch/pull/15239))
 - Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325))
 - Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895))
 - Star tree mapping changes ([#14605](https://github.com/opensearch-project/OpenSearch/pull/14605))
@@ -30,6 +31,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add concurrent search support for Derived Fields ([#15326](https://github.com/opensearch-project/OpenSearch/pull/15326))
 - [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343)))
 - Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630))
+- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))
+- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
+- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527))
+- [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410))
 - Optimize NodeIndicesStats output behind flag ([#14454](https://github.com/opensearch-project/OpenSearch/pull/14454))
 
 ### Dependencies
@@ -62,10 +67,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
 - Optimize regexp-based include/exclude on aggregations when pattern matches prefixes ([#14371](https://github.com/opensearch-project/OpenSearch/pull/14371))
 - Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238))
+- Remote publication using minimum node version for backward compatibility ([#15216](https://github.com/opensearch-project/OpenSearch/pull/15216))
+
 
 ### Deprecated
 
 ### Removed
+- Remove some unused code in the search backpressure package ([#15518](https://github.com/opensearch-project/OpenSearch/pull/15518))
 
 ### Fixed
 - Fix constraint bug which allows more primary shards than average primary shards per index ([#14908](https://github.com/opensearch-project/OpenSearch/pull/14908))
@@ -79,6 +87,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix indexing error when flat_object field is explicitly null ([#15375](https://github.com/opensearch-project/OpenSearch/pull/15375))
 - Fix split response processor not included in allowlist ([#15393](https://github.com/opensearch-project/OpenSearch/pull/15393))
 - Fix unchecked cast in dynamic action map getter ([#15394](https://github.com/opensearch-project/OpenSearch/pull/15394))
+- Fix null values indexed as "null" strings in flat_object field ([#14069](https://github.com/opensearch-project/OpenSearch/pull/14069))
 
 ### Security
 

diff --git a/...mmon/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java b/...mmon/src/main/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessor.java
@@ -59,6 +59,7 @@ public class ApiAnnotationProcessor extends AbstractProcessor {
     private static final String OPENSEARCH_PACKAGE = "org.opensearch";
 
     private final Set<Element> reported = new HashSet<>();
+    private final Set<Element> validated = new HashSet<>();
     private final Set<AnnotatedConstruct> processed = new HashSet<>();
     private Kind reportFailureAs = Kind.ERROR;
 
@@ -85,6 +86,8 @@ public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment
         );
 
         for (var element : elements) {
+            validate(element);
+
             if (!checkPackage(element)) {
                 continue;
             }
@@ -100,6 +103,64 @@ public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment
         return false;
     }
 
+    private void validate(Element element) {
+        // The element was validated already
+        if (validated.contains(element)) {
+            return;
+        }
+
+        validated.add(element);
+
+        final PublicApi publicApi = element.getAnnotation(PublicApi.class);
+        if (publicApi != null) {
+            if (!validateVersion(publicApi.since())) {
+                processingEnv.getMessager()
+                    .printMessage(
+                        reportFailureAs,
+                        "The type " + element + " has @PublicApi annotation with unparseable OpenSearch version: " + publicApi.since()
+                    );
+            }
+        }
+
+        final DeprecatedApi deprecatedApi = element.getAnnotation(DeprecatedApi.class);
+        if (deprecatedApi != null) {
+            if (!validateVersion(deprecatedApi.since())) {
+                processingEnv.getMessager()
+                    .printMessage(
+                        reportFailureAs,
+                        "The type "
+                            + element
+                            + " has @DeprecatedApi annotation with unparseable OpenSearch version: "
+                            + deprecatedApi.since()
+                    );
+            }
+        }
+    }
+
+    private boolean validateVersion(String version) {
+        String[] parts = version.split("[.-]");
+        if (parts.length < 3 || parts.length > 4) {
+            return false;
+        }
+
+        int major = Integer.parseInt(parts[0]);
+        if (major > 3 || major < 0) {
+            return false;
+        }
+
+        int minor = Integer.parseInt(parts[1]);
+        if (minor < 0) {
+            return false;
+        }
+
+        int patch = Integer.parseInt(parts[2]);
+        if (patch < 0) {
+            return false;
+        }
+
+        return true;
+    }
+
     /**
      * Check top level executable element
      * @param executable top level executable element

diff --git a/...src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java b/...src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java
@@ -486,4 +486,35 @@ public void testPublicApiConstructorAnnotatedInternalApi() {
 
         assertThat(failure.diagnotics(), not(hasItem(matching(Diagnostic.Kind.ERROR))));
     }
+
+    public void testPublicApiUnparseableVersion() {
+        final CompilerResult result = compile("PublicApiAnnotatedUnparseable.java");
+        assertThat(result, instanceOf(Failure.class));
+
+        final Failure failure = (Failure) result;
+        assertThat(failure.diagnotics(), hasSize(3));
+
+        assertThat(
+            failure.diagnotics(),
+            hasItem(
+                matching(
+                    Diagnostic.Kind.ERROR,
+                    containsString(
+                        "The type org.opensearch.common.annotation.processor.PublicApiAnnotatedUnparseable has @PublicApi annotation with unparseable OpenSearch version: 2.x"
+                    )
+                )
+            )
+        );
+    }
+
+    public void testPublicApiWithDeprecatedApiMethod() {
+        final CompilerResult result = compile("PublicApiWithDeprecatedApiMethod.java");
+        assertThat(result, instanceOf(Failure.class));
+
+        final Failure failure = (Failure) result;
+        assertThat(failure.diagnotics(), hasSize(2));
+
+        assertThat(failure.diagnotics(), not(hasItem(matching(Diagnostic.Kind.ERROR))));
+    }
+
 }
diff --git a/...t/resources/org/opensearch/common/annotation/processor/PublicApiAnnotatedUnparseable.java b/...t/resources/org/opensearch/common/annotation/processor/PublicApiAnnotatedUnparseable.java
@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.annotation.processor;
+
+import org.opensearch.common.annotation.PublicApi;
+
+@PublicApi(since = "2.x")
+public class PublicApiAnnotatedUnparseable {
+
+}
diff --git a/...esources/org/opensearch/common/annotation/processor/PublicApiWithDeprecatedApiMethod.java b/...esources/org/opensearch/common/annotation/processor/PublicApiWithDeprecatedApiMethod.java
@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.annotation.processor;
+
+import org.opensearch.common.annotation.DeprecatedApi;
+import org.opensearch.common.annotation.PublicApi;
+
+@PublicApi(since = "1.0.0")
+public class PublicApiWithDeprecatedApiMethod {
+    @DeprecatedApi(since = "0.1.0")
+    public void method() {
+
+    }
+}
diff --git a/...t-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4BadRequestIT.java b/...t-reactor-netty4/src/javaRestTest/java/org/opensearch/rest/ReactorNetty4BadRequestIT.java
@@ -112,4 +112,16 @@ public void testInvalidHeaderValue() throws IOException {
         assertThat(map.get("type"), equalTo("content_type_header_exception"));
         assertThat(map.get("reason"), equalTo("java.lang.IllegalArgumentException: invalid Content-Type header []"));
     }
+
+    public void testUnsupportedContentType() throws IOException {
+        final Request request = new Request("POST", "/_bulk/stream");
+        final RequestOptions.Builder options = request.getOptions().toBuilder();
+        request.setOptions(options);
+        final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
+        final Response response = e.getResponse();
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(406));
+        final ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        final String error = objectPath.evaluate("error");
+        assertThat(error, equalTo("Content-Type header [] is not supported"));
+    }
 }