[Dummy] Trial: bwc test enablement logic

.github/workflows/test_bwc.yml

@@ -0,0 +1,60 @@
+name: Test Custom Codecs BWC
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  Build-custom-codecs-linux:
+    needs: Get-CI-Image-Tag
+    strategy:
+      matrix:
+        java: [11,17,21]
+      fail-fast: false
+
+    name: Test Custom Codecs BWC
+    runs-on: ubuntu-latest
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
+
+    steps:
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: ${{ matrix.java }}
+
+      # custom-codecs
+      - name: Checkout Custom codecs
+        uses: actions/checkout@v3
+
+      - name: Assemble custom-codecs
+        run: |
+          plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'`
+          chown -R 1000:1000 `pwd`
+          echo plugin_version $plugin_version
+          su `id -un 1000` -c "./gradlew assemble"
+          echo "Creating ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/custom-codecs/$plugin_version ..."
+          su `id -un 1000` -c "mkdir -p ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version"
+          echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version ..."
+          ls ./build/distributions/
+          su `id -un 1000` -c "cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version"
+          echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version ..."
+          ls ./src/test/resources/org/opensearch/customcodecs/bwc/custom-codecs/$plugin_version
+      - name: Run Custom Codecs Backwards Compatibility Tests
+        run: |
+          echo "Running backwards compatibility tests ..."
+          chown -R 1000:1000 `pwd`
+          su `id -un 1000` -c "./gradlew bwcTestSuite -Dtests.security.manager=false --debug"

build.gradle

@@ -1,3 +1,6 @@
+import java.util.concurrent.Callable
+import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
+
 /*
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -15,6 +18,17 @@ buildscript {
     opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
     buildVersionQualifier = System.getProperty("build.version_qualifier", "")
+
+    bwcVersionShort = "2.11.0"
+    bwcVersion = bwcVersionShort + ".0"
+      //https://github.com/opensearch-project/custom-codecs/archive/refs/tags/2.11.0.0.zip
+//    bwcOpenSearchCustomCodecsDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' +
+//            'opensearch/plugins/custom-codecs-' + bwcVersion + '.zip'
+      //https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-custom-codecs/2.11.0.0/opensearch-custom-codecs-2.11.0.0.zip
+      bwcOpenSearchCustomCodecsDownload = 'https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-custom-codecs/' + bwcVersion + '/opensearch-custom-codecs-' + bwcVersion + '.zip'
+    baseName = "customCodecsBwcCluster"
+    bwcFilePath = "src/test/resources/org/opensearch/customcodecs/bwc/"
+    bwcCustomCodecsPath = bwcFilePath + "custom-codecs/"
   }
 
   repositories {
@@ -185,13 +199,179 @@ integTest {
     systemProperty "https", System.getProperty("https")
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
+
+    if (System.getProperty("tests.rest.bwcsuite") == null) {
+        filter {
+            excludeTestsMatching "org.opensearch.index.codecs.customcodecs.bwc.*IT"
+        }
+    }
 }
 
 testClusters.integTest {
     testDistribution = "ARCHIVE"
     plugin(project.tasks.bundlePlugin.archiveFile)
 }
 
+2.times {i ->
+    testClusters {
+        "${baseName}$i" {
+            testDistribution = "ARCHIVE"
+            versions = [bwcVersionShort, opensearch_version]
+            numberOfNodes = 3
+            plugin(provider(new Callable<RegularFile>(){
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            if (new File("$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion").exists()) {
+                                project.delete(files("$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion"))
+                            }
+                            project.mkdir bwcCustomCodecsPath + bwcVersion
+                            ant.get(src: bwcOpenSearchCustomCodecsDownload,
+                                    dest: "$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion",
+                                    httpusecaches: false)
+                            return fileTree("$project.rootDir/$bwcFilePath/custom-codecs/$bwcVersion").getSingleFile()
+                        }
+                    }
+                }
+            }))
+            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+            setting 'http.content_type.required', 'true'
+        }
+    }
+}
+
+String opensearchCustomCodecsPlugin = "opensearch-custom-codecs-" + project.version + ".zip"
+List<Provider<RegularFile>> plugins = [
+        provider(new Callable<RegularFile>() {
+            @Override
+            RegularFile call() throws Exception {
+                return new RegularFile() {
+                    @Override
+                    File getAsFile() {
+                        project.mkdir "$bwcFilePath/$project.version"
+                        copy {
+                            from "$buildDir/distributions/$opensearchCustomCodecsPlugin"
+                            into "$bwcFilePath/$project.version"
+                        }
+                        return fileTree(bwcFilePath + project.version).getSingleFile()
+                    }
+                }
+            }
+        })
+]
+
+// Creates 2 test clusters with 3 nodes of the old version.
+2.times {i ->
+    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
+        useCluster testClusters."${baseName}$i"
+        filter {
+            includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*"
+        }
+        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
+        systemProperty 'tests.rest.bwcsuite_round', 'old'
+        systemProperty 'tests.plugin_bwc_version', bwcVersion
+        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
+        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
+    }
+}
+
+// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
+// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
+// This is also used as a one third upgraded cluster for a rolling upgrade.
+task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
+    useCluster testClusters."${baseName}0"
+    dependsOn "${baseName}#oldVersionClusterTask0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'first'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
+// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
+// This is used for rolling upgrade.
+task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#mixedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'second'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
+// This results in a fully upgraded cluster.
+// This is used for rolling upgrade.
+task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*"
+    }
+    mustRunAfter "${baseName}#mixedClusterTask"
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'third'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
+// at the same time resulting in a fully upgraded cluster.
+task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#oldVersionClusterTask1"
+    useCluster testClusters."${baseName}1"
+    doFirst {
+        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.index.codec.customcodecs.bwc.*"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
+}
+
+// A bwc test suite which runs all the bwc tasks combined.
+task bwcTestSuite(type: StandaloneRestIntegTestTask) {
+    exclude '**/*Test*'
+    exclude '**/*IT*'
+    dependsOn tasks.named("${baseName}#mixedClusterTask")
+    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
+}
+
+run {
+    doFirst {
+        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
+        // not being written, the waitForAllConditions ensures it's written
+        getClusters().forEach { cluster ->
+            cluster.waitForAllConditions()
+        }
+    }
+    useCluster testClusters.integTest
+}
+
 tasks.withType(PublishToMavenRepository) {
     def predicate = provider {
         publication.name == "pluginZip"

scripts/bwctest.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+function usage() {
+  echo ""
+    echo "This script is used to run Backwards Compatibility tests"
+    echo "--------------------------------------------------------------------------"
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Required arguments:"
+    echo "None"
+    echo ""
+    echo -e "-h\tPrint this message."
+    echo "--------------------------------------------------------------------------"
+}
+
+while getopts ":h" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${OPTARG}"
+            exit 1
+            ;;
+    esac
+done
+
+./gradlew bwcTestSuite -Dtests.security.manager=false

src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java

@@ -47,11 +47,11 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin
         int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         if (mapperService == null) {
-            codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel));
+            codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel));
+            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel));
         } else {
-            codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel));
-            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel));
+            codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel));
+            codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel));
         }
         this.codecs = codecs.immutableMap();
     }

src/main/java/org/opensearch/index/codec/customcodecs/Lucene95CustomCodec.java → src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java

@@ -11,21 +11,21 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
 import org.opensearch.index.mapper.MapperService;
 
-import java.util.Collections;
 import java.util.Set;
 
 /**
  *
  * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec.
  * Supports two modes zstd and zstd_no_dict.
+ * Uses Lucene99 as the delegate codec
  *
  * @opensearch.internal
  */
-public abstract class Lucene95CustomCodec extends FilterCodec {
+public abstract class Lucene99CustomCodec extends FilterCodec {
 
     /** Default compression level used for compression */
     public static final int DEFAULT_COMPRESSION_LEVEL = 3;
@@ -35,17 +35,11 @@ public enum Mode {
         /**
          * ZStandard mode with dictionary
          */
-        ZSTD("ZSTD", Set.of("zstd")),
+        ZSTD("ZSTD99", Set.of("zstd")),
         /**
          * ZStandard mode without dictionary
          */
-        ZSTD_NO_DICT("ZSTDNODICT", Set.of("zstd_no_dict")),
-        /**
-         * Deprecated ZStandard mode, added for backward compatibility to support indices created in 2.9.0 where
-         * both ZSTD and ZSTD_NO_DICT used Lucene95CustomCodec underneath. This should not be used to
-         * create new indices.
-         */
-        ZSTD_DEPRECATED("Lucene95CustomCodec", Collections.emptySet());
+        ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict"));
 
         private final String codec;
         private final Set<String> aliases;
@@ -77,7 +71,7 @@ public Set<String> getAliases() {
      *
      * @param mode The compression codec (ZSTD or ZSTDNODICT).
      */
-    public Lucene95CustomCodec(Mode mode) {
+    public Lucene99CustomCodec(Mode mode) {
         this(mode, DEFAULT_COMPRESSION_LEVEL);
     }
 
@@ -89,9 +83,9 @@ public Lucene95CustomCodec(Mode mode) {
      * @param mode The compression codec (ZSTD or ZSTDNODICT).
      * @param compressionLevel The compression level.
      */
-    public Lucene95CustomCodec(Mode mode, int compressionLevel) {
-        super(mode.getCodec(), new Lucene95Codec());
-        this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel);
+    public Lucene99CustomCodec(Mode mode, int compressionLevel) {
+        super(mode.getCodec(), new Lucene99Codec());
+        this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel);
     }
 
     /**
@@ -104,9 +98,9 @@ public Lucene95CustomCodec(Mode mode, int compressionLevel) {
      * @param mapperService The mapper service.
      * @param logger The logger.
      */
-    public Lucene95CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
-        super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene95Codec.Mode.BEST_SPEED, mapperService, logger));
-        this.storedFieldsFormat = new Lucene95CustomStoredFieldsFormat(mode, compressionLevel);
+    public Lucene99CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
+        super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
+        this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel);
     }
 
     @Override