Merge branch 'main' into flush-user-cache

.github/actions/start-opensearch-with-one-plugin/action.yml

@@ -71,23 +71,22 @@ runs:
         'y' | .\opensearch-${{ inputs.opensearch-version }}-SNAPSHOT\bin\opensearch-plugin.bat install file:$(pwd)\${{ inputs.plugin-name }}.zip
       shell: pwsh
 
-    - name: Write password to initialAdminPassword location
-      run:
-        echo ${{ inputs.admin-password }} >> ./opensearch-${{ env.OPENSEARCH_VERSION }}-SNAPSHOT/config/initialAdminPassword.txt
-      shell: bash
-
     # Run any configuration scripts
     - name: Run Setup Script for Linux
       if: ${{ runner.os == 'Linux' && inputs.setup-script-name != '' }}
       run: |
         echo "running linux setup"
+        export OPENSEARCH_INITIAL_ADMIN_PASSWORD=${{ inputs.admin-password }}
         chmod +x ./${{ inputs.setup-script-name }}.sh
         ./${{ inputs.setup-script-name }}.sh
       shell: bash
 
     - name: Run Setup Script for Windows
       if: ${{ runner.os == 'Windows' && inputs.setup-script-name != '' }}
-      run: .\${{ inputs.setup-script-name }}.bat
+      run: |
+        echo "running windows setup"
+        $env:OPENSEARCH_INITIAL_ADMIN_PASSWORD="${{ inputs.admin-password }}"
+        .\${{ inputs.setup-script-name }}.bat
       shell: pwsh
 
     # Run OpenSearch

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       separateTestsNames: ${{ steps.set-matrix.outputs.separateTestsNames }}
     steps:
     - name: Set up JDK for build and test
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: 17
@@ -45,7 +45,7 @@ jobs:
 
     steps:
     - name: Set up JDK for build and test
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: ${{ matrix.jdk }}
@@ -74,7 +74,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           path: downloaded-artifacts
 
@@ -111,12 +111,12 @@ jobs:
       fail-fast: false
       matrix:
         jdk: [11, 17, 21]
-        platform: [ubuntu-latest] # Removed windows https://github.com/opensearch-project/security/issues/3423
+        platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
 
     steps:
     - name: Set up JDK for build and test
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: ${{ matrix.jdk }}
@@ -151,7 +151,7 @@ jobs:
 
     steps:
     - name: Set up JDK for build and test
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: ${{ matrix.jdk }}
@@ -169,7 +169,7 @@ jobs:
   backward-compatibility-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: 17
@@ -193,7 +193,7 @@ jobs:
     runs-on: ${{ matrix.platform }}
 
     steps:
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: ${{ matrix.jdk }}
@@ -214,22 +214,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: 11
-    - uses: github/codeql-action/init@v2
+    - uses: github/codeql-action/init@v3
       with:
         languages: java
     - run: ./gradlew clean assemble
-    - uses: github/codeql-action/analyze@v2
+    - uses: github/codeql-action/analyze@v3
 
   build-artifact-names:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: 11

.github/workflows/code-hygiene.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: 17
@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: 11
@@ -51,7 +51,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: 11

.github/workflows/integration-tests.yml

@@ -15,7 +15,7 @@ jobs:
         test-run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
 
     steps:
-    - uses: actions/setup-java@v3
+    - uses: actions/setup-java@v4
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: ${{ matrix.jdk }}

.github/workflows/maven-publish.yml

@@ -17,7 +17,7 @@ jobs:
       contents: write
 
     steps:
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: 11

.github/workflows/plugin_install.yml

@@ -20,7 +20,7 @@ jobs:
         uses: peternied/random-name@v1
 
       - name: Set up JDK
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: temurin # Temurin is a distribution of adoptium
           java-version: ${{ matrix.jdk }}

DEVELOPER_GUIDE.md

@@ -162,6 +162,18 @@ extension_hw_greet:
     - "hw-user"
 ```
 
+### Setting up password for demo admin user
+
+This step is a pre-requisite to installing demo configuration. You can pass the demo `admin` user password by exporting `OPENSEARCH_INITIAL_ADMIN_PASSWORD` variable with a password.
+```shell
+export OPENSEARCH_INITIAL_ADMIN_PASSWORD=<password>
+```
+
+**_Note:_** If no password is supplied, the installation will fail. The password supplied will also be tested for its strength and will be blocked if it is too simple. There is an option to skip this password validation by passing the `-t` option to the installation script. However, this should only be used for test environments.
+
+
+### Executing the demo installation script
+
 To install the demo certificates and default configuration, answer `y` to the first two questions and `n` to the last one. The log should look like below:
 
 ```bash
@@ -192,17 +204,17 @@ Detected OpenSearch Security Version: *
 "/Users/XXXXX/Test/opensearch-*/plugins/opensearch-security/tools/securityadmin.sh" -cd "/Users/XXXXX/Test/opensearch-*/config/opensearch-security/" -icl -key "/Users/XXXXX/Test/opensearch-*/config/kirk-key.pem" -cert "/Users/XXXXX/Test/opensearch-*/config/kirk.pem" -cacert "/Users/XXXXX/Test/opensearch-*/config/root-ca.pem" -nhnv
 ### or run ./securityadmin_demo.sh
 ### To use the Security Plugin ConfigurationGUI
-### To access your secured cluster open https://<hostname>:<HTTP port> and log in with admin/admin.
+### To access your secured cluster open https://<hostname>:<HTTP port> and log in with admin/<your-admin-password>.
 ### (Ignore the SSL certificate warning because we installed self-signed demo certificates)
 ```
 
 Now if we start our server again and try the original `curl localhost:9200`, it will fail.
-Try this command instead: `curl -XGET https://localhost:9200 -u 'admin:admin' --insecure`. It should succeed.
+Try this command instead: `curl -XGET https://localhost:9200 -u 'admin:<your-admin-password>' --insecure`. It should succeed.
 
 You can also make this call to return the authenticated user details:
 
 ```bash
-curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:admin' --insecure
+curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:<your-admin-password>' --insecure
 
 {
   "user": "User [name=admin, backend_roles=[admin], requestedTenant=null]",
@@ -232,11 +244,30 @@ curl -XGET https://localhost:9200/_plugins/_security/authinfo -u 'admin:admin' -
 
 Launch IntelliJ IDEA, choose **Project from Existing Sources**, and select directory with Gradle build script (`build.gradle`).
 
-## Running integration tests
+## Running tests
 
 Locally these can be run with `./gradlew test` with detailed results being available at `${project-root}/build/reports/tests/test/index.html`. You can also run tests through an IDEs JUnit test runner.
 
-Integration tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are available by going to the GitHub Actions workflow summary view and downloading the workflow run of the tests. If you see multiple tests listed with different JDK versions, you can download the version with whichever JDK you are interested in. After extracting the test file on your local machine, integration tests results can be found at `./tests/tests/index.html`.
+Tests are automatically run on all pull requests for all supported versions of the JDK. These must pass for change(s) to be merged. Detailed logs of these test results are available by going to the GitHub Actions workflow summary view and downloading the workflow run of the tests. If you see multiple tests listed with different JDK versions, you can download the version with whichever JDK you are interested in. After extracting the test file on your local machine, integration tests results can be found at `./tests/tests/index.html`.
+
+### Running an individual test multiple times
+
+This repo has a `@Repeat` annotation which you can import to annotate a test to run many times repeatedly. To use the annotation, add the following code to your test suite.
+
+```
+@Rule
+public RepeatRule repeatRule = new RepeatRule();
+
+@Test
+@Repeat(10)
+public void testMethod() {
+    ...
+}
+```
+
+## Running tests in the integrationTest package
+
+Tests in the integrationTest package can be run with `./gradlew integrationTest`.
 
 ### Bulk test runs
 

README.md

@@ -83,10 +83,6 @@ Run tests against local cluster:
 ```bash
 ./gradlew integTestRemote -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin -Dcommon_utils.version="2.2.0.0"
 ```
-OR
-```bash
-./scripts/integtest.sh
-```
 Note: To run against a remote cluster replace cluster-name and `localhost:9200` with the IPAddress:Port of that cluster.
 
 Build artifacts (zip, deb, rpm):

build.gradle

@@ -25,7 +25,7 @@ buildscript {
         opensearch_build = version_tokens[0] + '.0'
 
         common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT')
-        kafka_version  = '3.6.0'
+        kafka_version  = '3.6.1'
         apache_cxf_version = '4.0.3'
         open_saml_version = '4.3.0'
         one_login_java_saml = '2.9.0'
@@ -62,12 +62,12 @@ plugins {
     id 'idea'
     id 'jacoco'
     id 'maven-publish'
-    id 'com.diffplug.spotless' version '6.23.0'
+    id 'com.diffplug.spotless' version '6.23.3'
     id 'checkstyle'
-    id 'com.netflix.nebula.ospackage' version "11.5.0"
-    id "org.gradle.test-retry" version "1.5.6"
+    id 'com.netflix.nebula.ospackage' version "11.6.0"
+    id "org.gradle.test-retry" version "1.5.8"
     id 'eclipse'
-    id "com.github.spotbugs" version "5.2.4"
+    id "com.github.spotbugs" version "5.2.5"
     id "com.google.osdetector" version "1.7.3"
 }
 
@@ -256,6 +256,8 @@ test {
     jvmArgs += "-Xmx3072m"
     if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
         jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
+        // this is needed to reflect access system env map.
+        jvmArgs += "--add-opens=java.base/java.util=ALL-UNNAMED"
     }
     retry {
         failOnPassedAfterRetry = false
@@ -303,6 +305,8 @@ def setCommonTestConfig(Test task) {
     task.jvmArgs += "-Xmx3072m"
     if (JavaVersion.current() > JavaVersion.VERSION_1_8) {
         task.jvmArgs += "--add-opens=java.base/java.io=ALL-UNNAMED"
+        // this is needed to reflect access system env map.
+        task.jvmArgs += "--add-opens=java.base/java.util=ALL-UNNAMED"
     }
     task.retry {
         failOnPassedAfterRetry = false
@@ -490,13 +494,14 @@ configurations {
             force "org.apache.commons:commons-lang3:${versions.commonslang}"
 
             // for spotless transitive dependency CVE
-            force "org.eclipse.platform:org.eclipse.core.runtime:3.29.0"
+            force "org.eclipse.platform:org.eclipse.core.runtime:3.30.0"
 
             // For integrationTest
             force "org.apache.httpcomponents:httpclient:4.5.14"
             force "org.apache.httpcomponents:httpcore:4.4.16"
             force "com.google.errorprone:error_prone_annotations:2.23.0"
-            force "org.checkerframework:checker-qual:3.40.0"
+            force "org.checkerframework:checker-qual:3.42.0"
+            force "ch.qos.logback:logback-classic:1.2.13"
         }
     }
 
@@ -579,14 +584,14 @@ dependencies {
     implementation 'commons-cli:commons-cli:1.6.0'
     implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}"
     implementation 'org.ldaptive:ldaptive:1.2.3'
-    implementation 'com.nimbusds:nimbus-jose-jwt:9.37.1'
+    implementation 'com.nimbusds:nimbus-jose-jwt:9.37.3'
 
     //JWT
     implementation "io.jsonwebtoken:jjwt-api:${jjwt_version}"
     implementation "io.jsonwebtoken:jjwt-impl:${jjwt_version}"
     implementation "io.jsonwebtoken:jjwt-jackson:${jjwt_version}"
     // JSON patch
-    implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.14'
+    implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.16'
     implementation 'org.apache.commons:commons-collections4:4.4'
 
     //Password generation
@@ -609,7 +614,7 @@ dependencies {
     runtimeOnly 'jakarta.xml.bind:jakarta.xml.bind-api:4.0.1'
     runtimeOnly 'org.ow2.asm:asm:9.6'
 
-    testImplementation 'org.apache.camel:camel-xmlsecurity:3.21.2'
+    testImplementation 'org.apache.camel:camel-xmlsecurity:3.21.3'
 
     //OpenSAML
     implementation 'net.shibboleth.utilities:java-support:8.4.0'
@@ -648,7 +653,7 @@ dependencies {
     runtimeOnly 'org.apache.ws.xmlschema:xmlschema-core:2.3.1'
     runtimeOnly 'org.apache.santuario:xmlsec:2.3.4'
     runtimeOnly "com.github.luben:zstd-jni:${versions.zstd}"
-    runtimeOnly 'org.checkerframework:checker-qual:3.40.0'
+    runtimeOnly 'org.checkerframework:checker-qual:3.42.0'
     runtimeOnly "org.bouncycastle:bcpkix-jdk15to18:${versions.bouncycastle}"
     runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2'
 
@@ -677,11 +682,14 @@ dependencies {
     testImplementation "org.apache.kafka:kafka-group-coordinator:${kafka_version}"
     testImplementation "org.apache.kafka:kafka_2.13:${kafka_version}:test"
     testImplementation "org.apache.kafka:kafka-clients:${kafka_version}:test"
-    testImplementation 'commons-validator:commons-validator:1.7'
+    testImplementation 'commons-validator:commons-validator:1.8.0'
     testImplementation 'org.springframework.kafka:spring-kafka-test:2.9.13'
     testImplementation "org.springframework:spring-beans:${spring_version}"
     testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1'
     testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1'
+    testImplementation('org.awaitility:awaitility:4.2.0') {
+        exclude(group: 'org.hamcrest', module: 'hamcrest')
+    }
     // Only osx-x86_64, osx-aarch_64, linux-x86_64, linux-aarch_64, windows-x86_64 are available
     if (osdetector.classifier in ["osx-x86_64", "osx-aarch_64", "linux-x86_64", "linux-aarch_64", "windows-x86_64"]) {
         testImplementation "io.netty:netty-tcnative-classes:2.0.61.Final"
@@ -715,7 +723,7 @@ dependencies {
     integrationTestImplementation 'junit:junit:4.13.2'
     integrationTestImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}"
     integrationTestImplementation "org.opensearch.plugin:percolator-client:${opensearch_version}"
-    integrationTestImplementation 'commons-io:commons-io:2.15.0'
+    integrationTestImplementation 'commons-io:commons-io:2.15.1'
     integrationTestImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
     integrationTestImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"
     integrationTestImplementation 'org.hamcrest:hamcrest:2.2'