build.gradle

/*
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 *
 * Modifications Copyright OpenSearch Contributors. See
 * GitHub history for details.
 */


import javax.net.ssl.HostnameVerifier
import javax.net.ssl.HttpsURLConnection
import javax.net.ssl.SSLContext
import javax.net.ssl.SSLSession
import javax.net.ssl.TrustManager
import javax.net.ssl.X509TrustManager
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.security.GeneralSecurityException
import java.security.cert.X509Certificate
import java.util.concurrent.Callable
import java.util.function.Predicate
import java.util.concurrent.TimeUnit
import java.util.stream.Collectors
import org.opensearch.gradle.testclusters.OpenSearchCluster
import org.opensearch.gradle.test.RestIntegTestTask

buildscript {
    ext {
        isSnapshot = "true" == System.getProperty("build.snapshot", "false")
        opensearch_version = System.getProperty("opensearch.version", "1.2.0")
        // Taken from https://github.com/opensearch-project/alerting/blob/main/build.gradle#L33
        // 1.0.0 -> 1.0.0.0, and 1.0.0-SNAPSHOT -> 1.0.0.0-SNAPSHOT
        opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')

        // for bwc tests
        opensearch_previous_version = System.getProperty("bwc_older_version", "1.1.0")
        plugin_previous_version = opensearch_previous_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')

        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
        kotlin_version = System.getProperty("kotlin.version", "1.3.72")

    }

    repositories {
        mavenLocal()
        mavenCentral()
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
        maven { url "https://plugins.gradle.org/m2/" }
    }

    dependencies {
        classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${kotlin_version}"
        classpath "org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}"
        classpath "io.gitlab.arturbosch.detekt:detekt-gradle-plugin:1.0.0-RC15"
        //classpath "org.jacoco:org.jacoco.agent:0.8.5"
    }
}

plugins {
    id 'nebula.ospackage' version "8.3.0"
    id "com.dorongold.task-tree" version "1.5"
}

allprojects {
    group = "org.opensearch"
    if (isSnapshot) {
        version += "-SNAPSHOT"
    }
}

apply plugin: 'java'
//apply plugin: 'jacoco'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.testclusters'
apply plugin: 'opensearch.rest-test'
apply plugin: 'org.jetbrains.kotlin.jvm'
apply plugin: 'org.jetbrains.kotlin.plugin.allopen'

forbiddenApisTest.ignoreFailures = true

configurations.all {
    if (it.state != Configuration.State.UNRESOLVED) return
    resolutionStrategy {
        force "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
        force "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
        force 'junit:junit:4.13.1'
        force 'commons-beanutils:commons-beanutils:1.9.4'
        force 'com.google.guava:guava:30.0-jre'
        force 'com.puppycrawl.tools:checkstyle:8.29'
        force 'commons-codec:commons-codec:1.13'
        force 'org.apache.httpcomponents:httpclient:4.5.13'
        force 'org.apache.httpcomponents:httpclient-osgi:4.5.13'
        force 'org.apache.httpcomponents.client5:httpclient5:5.0.3'
        force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3'
        force 'com.fasterxml.jackson.core:jackson-databind:2.10.4'
        force 'org.yaml:snakeyaml:1.26'
        force 'org.codehaus.plexus:plexus-utils:3.0.24'
    }
}

dependencies {
    compileOnly "org.opensearch:opensearch:${opensearch_version}"
    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk7"
    compile "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
    compile "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
    compile "org.jetbrains:annotations:13.0"
    compile "com.github.seancfoley:ipaddress:5.3.3"
    compile "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.5"
    compile "org.opensearch:common-utils:${common_utils_version}"

    testCompile "org.opensearch.test:framework:${opensearch_version}"
    testImplementation "org.assertj:assertj-core:3.17.2"
    testImplementation "org.opensearch.client:opensearch-rest-high-level-client:${opensearch_version}"
    testImplementation "org.jetbrains.kotlinx:kotlinx-coroutines-test:1.3.5"
    testCompile "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
    testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
}

repositories {
    mavenLocal()
    mavenCentral()
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    maven { url "https://plugins.gradle.org/m2/" }
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    jcenter()
}

compileKotlin {
    kotlinOptions {
        // This should be 11, but the OpenSearch logger usage checker tool doesn't like classes > 1.8
        jvmTarget = "1.8"
        freeCompilerArgs = ['-Xjsr305=strict'] // Handle OpenSearch @Nullable annotation correctly
    }
}

compileTestKotlin {
    kotlinOptions {
        jvmTarget = "1.8"
        freeCompilerArgs = ['-Xjsr305=strict']
    }
}

ext {
    licenseFile = rootProject.file('LICENSE')
    noticeFile = rootProject.file('NOTICE')
}

opensearchplugin {
    name = project.name
    description = "OpenSearch Cross Cluster Replication Plugin"
    classname = "org.opensearch.replication.ReplicationPlugin"
}

java {
    if (!isSnapshot) {
        withJavadocJar()
        withSourcesJar()
    }
}

javadoc.enabled = false
licenseHeaders.enabled = false
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
validateNebulaPom.enabled = false
loggerUsageCheck.enabled = false

test {
    systemProperty 'tests.security.manager', 'false'
    if (System.getProperty("tests.debug") == "true") {
        debug true
        debugOptions {
            port = 8000
            suspend = false
        }
    }
}

// Setting RunTask.debug = true configures the JVM to use a debugger in listen mode (server=n,suspend=y).  This is a
// pain for multi node clusters since the node startup fails if it can't connect to a debugger. So instead we manually
// configure the debugger in attach mode (server=y) so that we can attach to a specific node after it has been started.
static String getDebugJvmArgs(int debugPort) {
    return " -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=${debugPort}"
}

def securityPluginFile = new Callable<RegularFile>() {
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    return fileTree("$projectDir/src/test/resources/security/plugin/opensearch-security-${opensearch_build}.zip").getSingleFile()
                }
            }
        }
    }

// Clone of WaitForHttpResource with updated code to support Cross cluster usecase
class CrossClusterWaitForHttpResource {

    private URL url;
    private String username;
    private String password;
    Set<Integer> validResponseCodes = Collections.singleton(200);

    CrossClusterWaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
        this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
    }

    CrossClusterWaitForHttpResource(URL url) {
        this.url = url;
    }

    boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException {
        final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs);
        final long sleep = 100;

        IOException failure = null;
        while (true) {
            try {
                checkResource();
                return true;
            } catch (IOException e) {
                failure = e;
            }
            if (System.nanoTime() < waitUntil) {
                Thread.sleep(sleep);
            } else {
                throw failure;
            }
        }
    }

    void setUsername(String username) {
        this.username = username;
    }

    void setPassword(String password) {
        this.password = password;
    }

    void checkResource() throws IOException {
        final HttpURLConnection connection = buildConnection()
        connection.connect();
        final Integer response = connection.getResponseCode();
        if (validResponseCodes.contains(response)) {
            return;
        } else {
            throw new IOException(response + " " + connection.getResponseMessage());
        }
    }

    HttpURLConnection buildConnection() throws IOException {
        final HttpURLConnection connection = (HttpURLConnection) this.@url.openConnection();

        if (connection instanceof HttpsURLConnection) {
            TrustManager[] trustAllCerts = [ new X509TrustManager() {
                    X509Certificate[] getAcceptedIssuers() {
                        return null;
                    }

                    void checkClientTrusted(X509Certificate[] certs, String authType) {
                    }

                    void checkServerTrusted(X509Certificate[] certs, String authType) {
                    }
                }
            ] as TrustManager[];
            SSLContext sc = SSLContext.getInstance("SSL");
            sc.init(null, trustAllCerts, new java.security.SecureRandom());
            connection.setSSLSocketFactory(sc.getSocketFactory());
            // Create all-trusting host name verifier
            HostnameVerifier allHostsValid = new HostnameVerifier() {
                boolean verify(String hostname, SSLSession session) {
                    return true;
                }
            };
            // Install the all-trusting host verifier
            connection.setHostnameVerifier(allHostsValid);
        }

        configureBasicAuth(connection);
        connection.setRequestMethod("GET");
        return connection;
    }

    void configureBasicAuth(HttpURLConnection connection) {
        if (username != null) {
            if (password == null) {
                throw new IllegalStateException("Basic Auth user [" + username + "] has been set, but no password has been configured");
            }
            connection.setRequestProperty(
                    "Authorization",
                    "Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8))
            );
        }
    }

}

/*
* To run security tests
*/
def securityEnabled = findProperty("security") == "true"

File repo = file("$buildDir/testclusters/repo")
def _numNodes = findProperty('numNodes') as Integer ?: 2
testClusters {
    leaderCluster {
        plugin(project.tasks.bundlePlugin.archiveFile)
        if(securityEnabled) {
            plugin(provider(securityPluginFile))
            cliSetup("opensearch-security/install_demo_configuration.sh", "-y")
        }
        int debugPort = 5005
        testDistribution = "INTEG_TEST"
        if (_numNodes > 1) numberOfNodes = _numNodes
        //numberOfNodes = 3
        setting 'path.repo', repo.absolutePath
        if(_numNodes == 1) jvmArgs "${-> getDebugJvmArgs(debugPort++)}"
    }
    followCluster {
        testDistribution = "INTEG_TEST"
        plugin(project.tasks.bundlePlugin.archiveFile)
        if(securityEnabled) {
            plugin(provider(securityPluginFile))
            cliSetup("opensearch-security/install_demo_configuration.sh", "-y")
        }
        int debugPort = 5010
        if (_numNodes > 1) numberOfNodes = _numNodes
        //numberOfNodes = 3
        setting 'path.repo', repo.absolutePath
        if(_numNodes == 1) jvmArgs "${-> getDebugJvmArgs(debugPort++)}"
    }
}

def configureCluster(OpenSearchCluster cluster, Boolean securityEnabled) {
    // clear existing health checks as we will need custom handling based on
    // security plugin installation
    cluster.@waitConditions.clear()
    String unicastUris = cluster.nodes.stream().flatMap { node ->
        node.getAllTransportPortURI().stream()
    }.collect(Collectors.joining("\n"))
    // Manually write the unicast hosts as we are not depending on the internal method
    cluster.nodes.forEach {node ->
        try {
            Files.write(node.getConfigDir().resolve("unicast_hosts.txt"), unicastUris.getBytes(StandardCharsets.UTF_8));
        } catch (IOException e) {
            throw new java.io.UncheckedIOException("Failed to write unicast_hosts for " + this, e);
        }
    }

    // Health check based on security plugin installation
    Predicate pred = { OpenSearchCluster c ->
        String protocol = "http"
        if(securityEnabled && !c.name.equalsIgnoreCase("integTest")) {
            protocol = "https"
        }
        CrossClusterWaitForHttpResource wait = new CrossClusterWaitForHttpResource(protocol, cluster.getFirstNode().getHttpSocketURI(), cluster.nodes.size())
        wait.setUsername("admin")
        wait.setPassword("admin")
        return wait.wait(500)
    }

    cluster.@waitConditions.put("cluster health yellow", pred)
    cluster.waitForAllConditions()
}

integTest {
    useCluster testClusters.leaderCluster
    useCluster testClusters.followCluster
    // We skip BWC test here as those get run as part of separate target `bwcTestSuite`.
    filter {
        excludeTestsMatching "org.opensearch.replication.bwc.*IT"
    }
    doFirst {
        getClusters().forEach { cluster ->
            String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllTransportPortURI().stream()
            }.collect(Collectors.joining(","))
            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllHttpSocketURI().stream()
            }.collect(Collectors.joining(","))

            systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
            configureCluster(cluster, securityEnabled)
        }
    }
    systemProperty "build.dir", "${buildDir}"
    systemProperty "java.security.policy", "file://${projectDir}/src/test/resources/plugin-security.policy"
}

run {
    useCluster testClusters.leaderCluster
    useCluster testClusters.followCluster
    doFirst {
        getClusters().forEach { cluster ->
            configureCluster(cluster, securityEnabled)
        }
    }
}

task initializeSecurityIndex {
    doLast {
        exec {
            executable "src/test/resources/security/scripts/SecurityAdminWrapper.sh"
            args "${buildDir}"
        }
    }
}

testingConventions {
    naming {
        IT {
            baseClass 'org.opensearch.replication.MultiClusterRestTestCase'
        }
    }
}

task release {
    dependsOn 'build'
}

//========================== BWC Test setup ======================================================

// TODO: Get from repo after its available: https://github.com/opensearch-project/opensearch-build/issues/184
String repl_old_zip = "src/test/resources/replication/opensearch-cross-cluster-replication-${plugin_previous_version}.zip"

def securityPluginOld = new Callable<RegularFile>() {
    @Override
    RegularFile call() throws Exception {
        return new RegularFile() {
            @Override
            File getAsFile() {
                return fileTree("$projectDir/src/test/resources/security/plugin/opensearch-security-${plugin_previous_version}.zip").getSingleFile()
            }
        }
    }
}

// We maintain 2 set of clusters here. One for full cluster restart and one for rolling restart + mixed cluster.
List<String> clusters = ["bwcLeader0", "bwcFollower0", "bwcLeader1", "bwcFollower1"]
// TODO: Make BWC test work with security plugin
clusters.each { name ->
    testClusters {
        "$name" {
            versions = [opensearch_previous_version, opensearch_version]
            plugin(provider(new Callable<RegularFile>() {
                @Override
                RegularFile call() throws Exception {
                    return new RegularFile() {
                        @Override
                        File getAsFile() {
                            return fileTree(repl_old_zip).getSingleFile()
                        }
                    }
                }
            }))

            if (securityEnabled) {
                plugin(provider(securityPluginOld))
                cliSetup("opensearch-security/install_demo_configuration.sh", "-y")
            }
            // Currently fetching the ARCHIVE distribution fails on mac as it tries to fetch the Mac specific "DARWIN" distribution
            // for Opensearch which is not publish yet. Changing this to INTEG_TEST to make it work on mac.
            if (System.getProperty("os.name").startsWith("Mac")) {
                testDistribution = "INTEG_TEST"
            } else {
                testDistribution = "ARCHIVE"
            }

            if (_numNodes != 3) numberOfNodes = 3
            setting 'path.repo', repo.absolutePath
        }
    }
}
List<Provider<RegularFile>> replPluginProvider = [
    provider(new Callable<RegularFile>(){
        @Override
        RegularFile call() throws Exception {
            return new RegularFile() {
                @Override
                File getAsFile() {
                    return fileTree("$projectDir/build/distributions/opensearch-cross-cluster-replication-${opensearch_build}.zip").getSingleFile()
                }
            }
        }
    })
]
if(securityEnabled) {
    replPluginProvider.append(provider(securityPluginFile))
}

/*
    Sets up and runs sanity test on older version clusters. We maintain 2 set of tasks here with.
    One for full cluster restart and one for rolling restart + mixed cluster.
 */
2.times { i ->
    task "oldVersionClusterTask$i" (type: RestIntegTestTask) {
        useCluster testClusters."bwcLeader$i"
        useCluster testClusters."bwcFollower$i"
        doFirst {
            getClusters().forEach { cluster ->
                String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
                    node.getAllTransportPortURI().stream()
                }.collect(Collectors.joining(","))
                String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
                    node.getAllHttpSocketURI().stream()
                }.collect(Collectors.joining(","))

                systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
                systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
                systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
                configureCluster(cluster, securityEnabled)
            }
        }
        systemProperty "build.dir", "${buildDir}"
        systemProperty "java.security.policy", "file://${projectDir}/src/test/resources/plugin-security.policy"
        nonInputProperties.systemProperty('tests.cluster_suffix', i)
        nonInputProperties.systemProperty('tests.bwcTask', "oldVersionClusterTask")
        filter {
            setIncludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT")
        }
    }
}

/*
    Can be executed with `./gradlew mixedClusterTask`

    Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
    This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
    This is also used as a one third upgraded cluster for a rolling upgrade.
 */
task "mixedClusterTask"(type: RestIntegTestTask) {
    dependsOn "oldVersionClusterTask0"
    useCluster testClusters.bwcLeader0
    useCluster testClusters.bwcFollower0

    doFirst {
        testClusters.bwcLeader0.upgradeNodeAndPluginToNextVersion(replPluginProvider)
        testClusters.bwcFollower0.upgradeNodeAndPluginToNextVersion(replPluginProvider)
        getClusters().forEach { cluster ->
            String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllTransportPortURI().stream()
            }.collect(Collectors.joining(","))
            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllHttpSocketURI().stream()
            }.collect(Collectors.joining(","))

            systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
            configureCluster(cluster, securityEnabled)
        }
    }
    nonInputProperties.systemProperty('tests.bwcTask', "mixedClusterTask")
    nonInputProperties.systemProperty('tests.cluster_suffix', "0")
    filter {
        setIncludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT")
    }
}

// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "twoThirdsUpgradedClusterTask"(type: RestIntegTestTask) {
    dependsOn "mixedClusterTask"
    useCluster testClusters.bwcLeader0
    useCluster testClusters.bwcFollower0
    doFirst {
        testClusters.bwcLeader0.upgradeNodeAndPluginToNextVersion(replPluginProvider)
        testClusters.bwcFollower0.upgradeNodeAndPluginToNextVersion(replPluginProvider)
        getClusters().forEach { cluster ->
            String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllTransportPortURI().stream()
            }.collect(Collectors.joining(","))
            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllHttpSocketURI().stream()
            }.collect(Collectors.joining(","))

            systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
            configureCluster(cluster, securityEnabled)
        }
    }
    nonInputProperties.systemProperty('tests.bwcTask', "twoThirdsUpgradedClusterTask")
    nonInputProperties.systemProperty('tests.cluster_suffix', "0")
    filter {
        setIncludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT")
    }
}

/*
    Can be executed with `./gradlew rollingUpgradeClusterTask`

    Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
    This results in a fully upgraded cluster.
    This is used for rolling upgrade.
*/
task "rollingUpgradeClusterTask"(type: RestIntegTestTask) {
    dependsOn "twoThirdsUpgradedClusterTask"
    useCluster testClusters.bwcLeader0
    useCluster testClusters.bwcFollower0
    doFirst {
        testClusters.bwcLeader0.upgradeNodeAndPluginToNextVersion(replPluginProvider)
        testClusters.bwcFollower0.upgradeNodeAndPluginToNextVersion(replPluginProvider)
        getClusters().forEach { cluster ->
            String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllTransportPortURI().stream()
            }.collect(Collectors.joining(","))
            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllHttpSocketURI().stream()
            }.collect(Collectors.joining(","))

            systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
            configureCluster(cluster, securityEnabled)
        }
    }
    nonInputProperties.systemProperty('tests.cluster_suffix', "0")
    nonInputProperties.systemProperty('tests.bwcTask', "rollingUpgradeClusterTask")
    filter {
        setIncludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT")
    }
}

/*
    Can be executed with `./gradlew fullRestartClusterTask`

    Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
    at the same time resulting in a fully upgraded cluster.
*/
task "fullRestartClusterTask"(type: RestIntegTestTask) {
    dependsOn "oldVersionClusterTask1"
    useCluster testClusters.bwcLeader1
    useCluster testClusters.bwcFollower1
    doFirst {
        testClusters.bwcLeader1.upgradeAllNodesAndPluginsToNextVersion(replPluginProvider)
        testClusters.bwcFollower1.upgradeAllNodesAndPluginsToNextVersion(replPluginProvider)
        getClusters().forEach { cluster ->
            String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllTransportPortURI().stream()
            }.collect(Collectors.joining(","))
            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllHttpSocketURI().stream()
            }.collect(Collectors.joining(","))

            systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
            configureCluster(cluster, securityEnabled)
        }
    }
    nonInputProperties.systemProperty('tests.cluster_suffix', "1")
    nonInputProperties.systemProperty('tests.bwcTask', "fullRestartClusterTask")
    filter {
        setIncludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT")
    }
}

/*
    Can be executed with `./gradlew bwcTestSuite`

    Executes all 3 following upgrade scenarios as part of this test suite.
    - mixed cluster:  oldVersionClusterTask0 --> mixedClusterTask
    - rolling restart:  oldVersionClusterTask0 --> mixedClusterTask -> twoThirdsUpgradedClusterTask -> rollingUpgradeClusterTask
    - full cluster restart:  oldVersionClusterTask1 --> fullRestartClusterTask
 */
task "bwcTestSuite"(type: RestIntegTestTask) {
    useCluster testClusters.bwcLeader0
    useCluster testClusters.bwcFollower0
    useCluster testClusters.bwcLeader1
    useCluster testClusters.bwcFollower1
    doFirst {
        getClusters().forEach { cluster ->
            String alltransportSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllTransportPortURI().stream()
            }.collect(Collectors.joining(","))
            String allHttpSocketURI = cluster.nodes.stream().flatMap { node ->
                node.getAllHttpSocketURI().stream()
            }.collect(Collectors.joining(","))

            systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}"
            systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}"
            configureCluster(cluster, securityEnabled)
        }
    }
    filter {
        setIncludePatterns("org.opensearch.replication.bwc.BackwardsCompatibilityIT")
    }
    nonInputProperties.systemProperty('tests.cluster_suffix', "1")
    nonInputProperties.systemProperty('tests.bwcTask', "bwcTestSuite")
    dependsOn tasks.named("mixedClusterTask")
    dependsOn tasks.named("rollingUpgradeClusterTask")
    dependsOn tasks.named("fullRestartClusterTask")
}