Merge branch 'opensearch-project:main' into rishma-dynamic-pruning-ca…

…rdinality-agg

.github/ISSUE_TEMPLATE/meta.yml

@@ -23,6 +23,36 @@ body:
       placeholder: Ex. - [ ] https://github.com/opensearch-project/security/issues/3888 Add views to the cluster metadata schema
     validations:
       required: true
-
-
-
+  - type: dropdown
+    attributes:
+      label: Related component
+      description: Choose a specific OpenSearch component your project belongs to.  If you are unsure of which component to select or if the component is not present, select "Other".
+      multiple: false
+      options:
+        - # Empty first option to force selection
+        - Build
+        - Clients
+        - Cluster Manager
+        - Extensions
+        - Indexing:Performance
+        - Indexing:Replication
+        - Indexing
+        - Libraries
+        - Other
+        - Plugins
+        - Search:Aggregations
+        - Search:Performance
+        - Search:Query Capabilities
+        - Search:Query Insights
+        - Search:Relevance
+        - Search:Remote Search
+        - Search:Resiliency
+        - Search:Searchable Snapshots
+        - Search
+        - Storage:Durability
+        - Storage:Performance
+        - Storage:Remote
+        - Storage:Snapshots
+        - Storage
+    validations:
+      required: true

CHANGELOG.md

@@ -48,6 +48,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
 - Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617))
 - Bump OpenTelemetry from 1.31.0 to 1.32.0 and OpenTelemetry Semconv from 1.21.0-alpha to 1.23.1-alpha ([#11305](https://github.com/opensearch-project/OpenSearch/pull/11305))
+- Bump `org.bouncycastle:bcprov-jdk15to18` to `org.bouncycastle:bcprov-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
+- Bump `org.bouncycastle:bcmail-jdk15to18` to `org.bouncycastle:bcmail-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
+- Bump `org.bouncycastle:bcpkix-jdk15to18` to `org.bouncycastle:bcpkix-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -112,6 +115,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix for deserilization bug in weighted round-robin metadata ([#11679](https://github.com/opensearch-project/OpenSearch/pull/11679))
 - Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298))
+- Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481))
 
 ### Security
 

buildSrc/version.properties

@@ -50,7 +50,7 @@ reactivestreams   = 1.0.4
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
-bouncycastle=1.76
+bouncycastle=1.77
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2

gradle/ide.gradle

@@ -82,6 +82,9 @@ if (System.getProperty('idea.active') == 'true') {
         runConfigurations {
           defaults(JUnit) {
             vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT'
+            if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
+              vmParameters += ' -Djava.security.manager=allow'
+            }
           }
         }
         copyright {

plugins/identity-shiro/build.gradle

@@ -28,7 +28,7 @@ dependencies {
 
   implementation 'org.passay:passay:1.6.3'
 
-  implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
+  implementation "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
 
   testImplementation project(path: ':modules:transport-netty4') // for http
   testImplementation project(path: ':plugins:transport-nio') // for http

plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+2cc971b6c20949c1ff98d1a4bc741ee848a09523

plugins/ingest-attachment/build.gradle

@@ -71,9 +71,9 @@ dependencies {
   api "org.apache.pdfbox:fontbox:${versions.pdfbox}"
   api "org.apache.pdfbox:jempbox:1.8.17"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.bouncycastle:bcmail-jdk15to18:${versions.bouncycastle}"
-  api "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
-  api "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
+  api "org.bouncycastle:bcmail-jdk18on:${versions.bouncycastle}"
+  api "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
+  api "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
   // OpenOffice
   api "org.apache.poi:poi-ooxml:${versions.poi}"
   api "org.apache.poi:poi:${versions.poi}"

plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+f2bb8aa55dc901ee8b8aae7d1007c03592d65e03

plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+ed953791ba0229747dd0fd9911e3d76a462acfd3

plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1

@@ -0,0 +1 @@
+2cc971b6c20949c1ff98d1a4bc741ee848a09523

server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java

@@ -59,19 +59,19 @@ public static Collection<Object[]> parameters() {
     @Override
     protected void setupSuiteScopeCluster() throws Exception {
         assertAcked(client().admin().indices().prepareCreate("idx").get());
+        expected.clear();
 
-        final int segmentCount = randomIntBetween(2, 10);
-        final Set<Long> longTerms = new HashSet();
+        final int repeat = randomIntBetween(2, 10);
+        final Set<Long> longTerms = new HashSet<>();
 
-        final Map<String, Integer> dateTerms = new HashMap<>();
-        for (int i = 0; i < segmentCount; i++) {
+        for (int i = 0; i < repeat; i++) {
             final List<IndexRequestBuilder> indexRequests = new ArrayList<>();
 
             long longTerm;
             do {
-                longTerm = randomInt(segmentCount * 2);
+                longTerm = randomInt(repeat * 2);
             } while (!longTerms.add(longTerm));
-            ZonedDateTime time = ZonedDateTime.of(2024, 1, ((int) longTerm % 20) + 1, 0, 0, 0, 0, ZoneOffset.UTC);
+            ZonedDateTime time = ZonedDateTime.of(2024, 1, ((int) longTerm) + 1, 0, 0, 0, 0, ZoneOffset.UTC);
             String dateTerm = DateFormatter.forPattern("yyyy-MM-dd").format(time);
 
             final int frequency = randomBoolean() ? 1 : randomIntBetween(2, 20);

server/src/main/java/org/opensearch/common/lucene/store/ByteArrayIndexInput.java

@@ -144,7 +144,7 @@ public long readLong(long pos) throws IOException {
     }
 
     private void validatePos(long pos, int len) throws EOFException {
-        if (pos < 0 || pos + len > length + offset) {
+        if (pos < 0 || pos + len > length) {
             throw new EOFException("seek past EOF");
         }
     }

server/src/test/java/org/opensearch/common/lucene/store/ByteArrayIndexInputTests.java

@@ -32,6 +32,8 @@
 
 package org.opensearch.common.lucene.store;
 
+import org.apache.lucene.store.IndexInput;
+
 import java.io.EOFException;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
@@ -153,4 +155,34 @@ public void testRandomAccessReads() throws IOException {
         // 10001001 00100101 10001001 00110000 11100111 00100100 10110001 00101110
         assertEquals(-8564288273245753042L, indexInput.readLong(1));
     }
+
+    public void testReadBytesWithSlice() throws IOException {
+        int inputLength = randomIntBetween(100, 1000);
+
+        byte[] input = randomUnicodeOfLength(inputLength).getBytes(StandardCharsets.UTF_8);
+        ByteArrayIndexInput indexInput = new ByteArrayIndexInput("test", input);
+
+        int sliceOffset = randomIntBetween(1, inputLength - 10);
+        int sliceLength = randomIntBetween(2, inputLength - sliceOffset);
+        IndexInput slice = indexInput.slice("slice", sliceOffset, sliceLength);
+
+        // read a byte from sliced index input and verify if the read value is correct
+        assertEquals(input[sliceOffset], slice.readByte());
+
+        // read few more bytes into a byte array
+        int bytesToRead = randomIntBetween(1, sliceLength - 1);
+        slice.readBytes(new byte[bytesToRead], 0, bytesToRead);
+
+        // now try to read beyond the boundary of the slice, but within the
+        // boundary of the original IndexInput. We've already read few bytes
+        // so this is expected to fail
+        assertThrows(EOFException.class, () -> slice.readBytes(new byte[sliceLength], 0, sliceLength));
+
+        // seek to EOF and then try to read
+        slice.seek(sliceLength);
+        assertThrows(EOFException.class, () -> slice.readBytes(new byte[1], 0, 1));
+
+        slice.close();
+        indexInput.close();
+    }
 }