build.gradle

/*
 * This file was generated by the Gradle 'init' task.
 *
 * This is a general purpose Gradle build.
 * Learn more about Gradle by exploring our samples at https://docs.gradle.org/7.5.1/samples
 * This project uses @Incubating APIs which are subject to change.
 */

import org.opensearch.gradle.test.RestIntegTestTask

import java.util.concurrent.Callable

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.pluginzip'
apply plugin: 'jacoco'
apply plugin: "com.diffplug.spotless"
apply plugin: 'io.freefair.lombok'

def pluginName = 'opensearch-neural-search'
def pluginDescription = 'A plugin that adds dense neural retrieval into the OpenSearch ecosytem'
def projectPath = 'org.opensearch'
def pathToPlugin = 'neuralsearch.plugin'
def pluginClassName = 'NeuralSearch'

tasks.register("preparePluginPathDirs") {
    mustRunAfter clean
    doLast {
        def newPath = pathToPlugin.replace(".", "/")
        mkdir "src/main/java/org/opensearch/$newPath"
        mkdir "src/test/java/org/opensearch/$newPath"
    }
}

publishing {
    repositories {
        maven {
            name = "Snapshots"
            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
            credentials {
                username "$System.env.SONATYPE_USERNAME"
                password "$System.env.SONATYPE_PASSWORD"
            }
        }
    }
    publications {
        pluginZip(MavenPublication) { publication ->
            pom {
                name = pluginName
                description = pluginDescription
                groupId = "org.opensearch.plugin"
                licenses {
                    license {
                        name = "The Apache License, Version 2.0"
                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
                    }
                }
                developers {
                    developer {
                        name = "OpenSearch"
                        url = "https://github.com/opensearch-project/neural-search"
                    }
                }
            }
        }
    }
}
opensearchplugin {
    name pluginName
    description pluginDescription
    classname "${projectPath}.${pathToPlugin}.${pluginClassName}"
    licenseFile rootProject.file('LICENSE')
    noticeFile rootProject.file('NOTICE')
    extendedPlugins = ['opensearch-knn']
}

dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false
loggerUsageCheck.enabled = false
// No need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false

buildscript {
    ext {
        opensearch_version = System.getProperty("opensearch.version", "2.11.1-SNAPSHOT")
        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
        version_tokens = opensearch_version.tokenize('-')
        opensearch_build = version_tokens[0] + '.0'
        plugin_no_snapshot = opensearch_build
        if (buildVersionQualifier) {
            opensearch_build += "-${buildVersionQualifier}"
            plugin_no_snapshot += "-${buildVersionQualifier}"
        }
        if (isSnapshot) {
            opensearch_build += "-SNAPSHOT"
        }
        opensearch_group = "org.opensearch"
        opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
    }

    repositories {
        mavenLocal()
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
    }

    dependencies {
        classpath "${opensearch_group}.gradle:build-tools:${opensearch_version}"
        classpath "com.diffplug.spotless:spotless-plugin-gradle:5.6.1"
        classpath "io.freefair.gradle:lombok-plugin:8.0.1"
    }
}

ext {
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
}

allprojects {
    group = opensearch_group
    version = "${opensearch_build}"
    targetCompatibility = JavaVersion.VERSION_11
    sourceCompatibility = JavaVersion.VERSION_11
}

repositories {
    mavenLocal()
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    mavenCentral()
    maven { url "https://plugins.gradle.org/m2/" }
}

configurations {
    zipArchive
}

def knnJarDirectory = "$buildDir/dependencies/opensearch-knn"

dependencies {
    api "org.opensearch:opensearch:${opensearch_version}"
    zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
    zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
    compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
    api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.10'
    // ml-common excluded reflection for runtime so we need to add it by ourselves.
    // https://github.com/opensearch-project/ml-commons/commit/464bfe34c66d7a729a00dd457f03587ea4e504d9
    // TODO: Remove following three lines of dependencies if ml-common include them in their jar
    runtimeOnly group: 'org.reflections', name: 'reflections', version: '0.9.12'
    runtimeOnly group: 'org.javassist', name: 'javassist', version: '3.29.2-GA'
    runtimeOnly group: 'org.opensearch', name: 'common-utils', version: "${opensearch_build}"
    runtimeOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
    runtimeOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
    runtimeOnly group: 'org.json', name: 'json', version: '20230227'
}

// In order to add the jar to the classpath, we need to unzip the
// k-NN zip and then copy it into a directory that we specify as a dependency.
task extractKnnJar(type: Copy) {
    mustRunAfter()
    from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-knn")}))
    into knnJarDirectory
}

project.tasks.delombok.dependsOn(extractKnnJar)

compileJava {
    dependsOn extractKnnJar
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}
compileTestJava {
    options.compilerArgs.addAll(["-processor", 'lombok.launch.AnnotationProcessorHider$AnnotationProcessor'])
}

def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()
def _numNodes = findProperty('numNodes') as Integer ?: 1

test {
    include '**/*Tests.class'
    systemProperty 'tests.security.manager', 'false'
}

// Setting up Integration Tests
task integTest(type: RestIntegTestTask) {
    description = "Run tests against a cluster"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    // allows integration test classes to access test resource from project root path
    systemProperty('project.root', project.rootDir.absolutePath)

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
    }
}

testClusters.integTest {
    testDistribution = "ARCHIVE"
    // Install K-NN/ml-commons plugins on the integTest cluster nodes
    configurations.zipArchive.asFileTree.each {
        plugin(provider(new Callable<RegularFile>(){
            @Override
            RegularFile call() throws Exception {
                return new RegularFile() {
                    @Override
                    File getAsFile() {
                        return it
                    }
                }
            }
        }))
    }

    // This installs our neural-search plugin into the testClusters
    plugin(project.tasks.bundlePlugin.archiveFile)
    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes
    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug opensearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("cluster.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=${debugPort}")
            debugPort += 1
        }
    }

    // Increase heap size from default of 512mb to 1gb. When heap size is 512mb, our integ tests sporadically fail due
    // to ml-commons memory circuit breaker exception
    jvmArgs("-Xms1g", "-Xmx1g")
}

// Remote Integration Tests
task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath

    systemProperty "https", System.getProperty("https")
    systemProperty "user", System.getProperty("user")
    systemProperty "password", System.getProperty("password")

    systemProperty 'cluster.number_of_nodes', "${_numNodes}"

    systemProperty 'tests.security.manager', 'false'

    // Run tests with remote cluster only if rest case is defined
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.neuralsearch.*IT"
        }
    }
}


run {
    useCluster testClusters.integTest
}

spotless {
    java {
        removeUnusedImports()
        importOrder 'java', 'javax', 'org', 'com'
        eclipse().configFile rootProject.file('formatterConfig.xml')
        trimTrailingWhitespace()
        endWithNewline()
    }
}

jacocoTestReport {
    dependsOn integTest, test
    reports {
        xml.required = true
        html.required = true
    }
}

check.dependsOn jacocoTestCoverageVerification
jacocoTestCoverageVerification.dependsOn jacocoTestReport


// updateVersion: Task to auto update version to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
        // String tokenization to support -SNAPSHOT
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
    }
}