Feature/merge (#199)

* Adding support for OS 1.x and 2.x

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Upgrading spark version

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fix versioning

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Upgrading gradle wrapper

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fix gradle errors

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

* Fix more build and test errors

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

---------

Signed-off-by: Harsha Vamsi Kalluri <harshavamsi096@gmail.com>

buildSrc/build.gradle

@@ -59,9 +59,12 @@ repositories {
     if (localRepo) {
         // For some reason the root dirs all point to the buildSrc folder. The local Repo will be one above that.
         flatDir { dirs new File(project.rootDir, "../localRepo") }
-    } else {
+    } else if (snapshot) {
         maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
     }
+    else {
+        maven { url "https://plugins.gradle.org/m2/" }
+    }
 }
 
 dependencies {

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy

@@ -66,7 +66,7 @@ class SparkYarnServiceDescriptor implements ServiceDescriptor {
 
     @Override
     Version defaultVersion() {
-        return new Version(3, 2, 3)
+        return new Version(3, 2, 4)
     }
 
     String hadoopVersionCompatibility() {

buildSrc/src/main/java/org/opensearch/hadoop/gradle/buildtools/info/GlobalBuildInfoPlugin.java

@@ -112,7 +112,7 @@ public void apply(Project project) {
             params.setRuntimeJavaHome(runtimeJavaHome);
             params.setRuntimeJavaVersion(determineJavaVersion("runtime java.home", runtimeJavaHome, minimumRuntimeVersion));
             params.setIsRuntimeJavaHomeSet(Jvm.current().getJavaHome().equals(runtimeJavaHome) == false);
-            JvmInstallationMetadata runtimeJdkMetaData = metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome).getLocation());
+            JvmInstallationMetadata runtimeJdkMetaData = metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome));
             params.setJavaVersions(getAvailableJavaVersions());
             params.setMinimumCompilerVersion(minimumCompilerVersion);
             params.setMinimumRuntimeVersion(minimumRuntimeVersion);
@@ -135,14 +135,15 @@ private void logGlobalBuildInfo() {
         final String osVersion = System.getProperty("os.version");
         final String osArch = System.getProperty("os.arch");
         final Jvm gradleJvm = Jvm.current();
-        JvmInstallationMetadata gradleJvmMetadata = metadataDetector.getMetadata(gradleJvm.getJavaHome());
+        JvmInstallationMetadata gradleJvmMetadata = metadataDetector.getMetadata(new InstallationLocation( gradleJvm.getJavaHome(), "current Java home"));
         final String gradleJvmVendorDetails = gradleJvmMetadata.getVendor().getDisplayName();
         LOGGER.quiet("=======================================");
         LOGGER.quiet("OpenSearch Build Hamster says Hello!");
         LOGGER.quiet("  Gradle Version        : " + GradleVersion.current().getVersion());
         LOGGER.quiet("  OS Info               : " + osName + " " + osVersion + " (" + osArch + ")");
         if (BuildParams.getIsRuntimeJavaHomeSet()) {
-            final String runtimeJvmVendorDetails = metadataDetector.getMetadata(BuildParams.getRuntimeJavaHome())
+            final String runtimeJvmVendorDetails = metadataDetector.getMetadata(new InstallationLocation(BuildParams.getRuntimeJavaHome(), 
+                    "current Java home"))
                 .getVendor()
                 .getDisplayName();
             LOGGER.quiet("  Runtime JDK Version   : " + BuildParams.getRuntimeJavaVersion() + " (" + runtimeJvmVendorDetails + ")");
@@ -160,7 +161,7 @@ private void logGlobalBuildInfo() {
 
     private JavaVersion determineJavaVersion(String description, File javaHome, JavaVersion requiredVersion) {
         InstallationLocation installation = getJavaInstallation(javaHome);
-        JavaVersion actualVersion = metadataDetector.getMetadata(installation.getLocation()).getLanguageVersion();
+        JavaVersion actualVersion = metadataDetector.getMetadata(installation).getLanguageVersion();
         if (actualVersion.isCompatibleWith(requiredVersion) == false) {
             throwInvalidJavaHomeException(
                 description,
@@ -194,7 +195,7 @@ private boolean isSameFile(File javaHome, InstallationLocation installationLocat
     private List<JavaHome> getAvailableJavaVersions() {
         return getAvailableJavaInstallationLocationSteam().map(installationLocation -> {
             File installationDir = installationLocation.getLocation();
-            JvmInstallationMetadata metadata = metadataDetector.getMetadata(installationDir);
+            JvmInstallationMetadata metadata = metadataDetector.getMetadata(installationLocation);
             int actualVersion = Integer.parseInt(metadata.getLanguageVersion().getMajorVersion());
             return JavaHome.of(actualVersion, providers.provider(() -> installationDir));
         }).collect(Collectors.toList());

buildSrc/src/main/java/org/opensearch/hadoop/gradle/scala/SparkVariantPlugin.java

@@ -64,7 +64,6 @@
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.api.tasks.scala.ScalaDoc;
 import org.gradle.api.tasks.testing.Test;
-import org.gradle.util.ConfigureUtil;
 
 import static org.gradle.api.plugins.JavaBasePlugin.DOCUMENTATION_GROUP;
 import static org.gradle.api.plugins.JavaBasePlugin.VERIFICATION_GROUP;
@@ -236,34 +235,18 @@ public SparkVariant addFeatureVariant(String variantName, String sparkVersion, S
             return variant;
         }
 
-        public void all(Closure configure) {
-            all(ConfigureUtil.configureUsing(configure));
-        }
-
         public void all(Action<SparkVariant> action) {
             variants.all(action);
         }
 
-        public void defaultVariant(Closure configure) {
-            defaultVariant(ConfigureUtil.configureUsing(configure));
-        }
-
         public void defaultVariant(Action<SparkVariant> action) {
             variants.matching(SparkVariant::isDefaultVariant).all(action);
         }
 
-        public void featureVariants(Closure configure) {
-            featureVariants(ConfigureUtil.configureUsing(configure));
-        }
-
         public void featureVariants(Action<SparkVariant> action) {
             variants.matching(element -> !element.isDefaultVariant()).all(action);
         }
 
-        public SparkVariant featureVariant(String featureVariant, Closure configure) {
-            return featureVariant(featureVariant, ConfigureUtil.configureUsing(configure));
-        }
-
         public SparkVariant featureVariant(String featureVariant, Action<SparkVariant> action) {
             return variants.getByName(featureVariant, action);
         }

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -205,6 +205,12 @@ set -- \
         org.gradle.wrapper.GradleWrapperMain \
         "$@"
 
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
 # Use "xargs" to parse quoted args.
 #
 # With -n1 it outputs one arg per line, with the quotes and backslashes removed.

gradlew.bat

@@ -14,7 +14,7 @@
 @rem limitations under the License.
 @rem
 
-@if "%DEBUG%" == "" @echo off
+@if "%DEBUG%"=="" @echo off
 @rem ##########################################################################
 @rem
 @rem  Gradle startup script for Windows
@@ -25,7 +25,7 @@
 if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
+if "%DIRNAME%"=="" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
@@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome
 
 set JAVA_EXE=java.exe
 %JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto execute
+if %ERRORLEVEL% equ 0 goto execute
 
 echo.
 echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
@@ -75,13 +75,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
 :end
 @rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
+if %ERRORLEVEL% equ 0 goto mainEnd
 
 :fail
 rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
 rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
+set EXIT_CODE=%ERRORLEVEL%
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
 
 :mainEnd
 if "%OS%"=="Windows_NT" endlocal

mr/src/main/java/org/opensearch/hadoop/rest/Resource.java

@@ -90,14 +90,6 @@ public Resource(Settings settings, boolean read) {
         int slash = res.indexOf("/");
         boolean typeExists = slash >= 0;
 
-        // Types can no longer the specified at all! Index names only!
-        if (typeExists) {
-            throw new OpenSearchHadoopIllegalArgumentException(String.format(
-                    "Detected type name in resource [%s]. Remove type name to continue.",
-                    resource
-            ));
-        }
-
         // Parse out the type if it exists and is valid.
         if (typeExists) {
             index = res.substring(0, slash);

mr/src/main/java/org/opensearch/hadoop/rest/RestClient.java

@@ -620,7 +620,13 @@ public boolean isAlias(String query) {
     public void putMapping(String index, String type, byte[] bytes) {
         // create index first (if needed) - it might return 403/404
         touch(index);
-        execute(PUT, index + "/_mapping", new BytesArray(bytes));
+        ClusterInfo clusterInfo = mainInfo();
+        if (clusterInfo.getMajorVersion().before(OpenSearchMajorVersion.V_2_X)) {
+            execute(PUT, index + "/_mapping/" + type + "?include_type_name=true", new BytesArray(bytes));
+        }
+        else {
+            execute(PUT, index + "/_mapping", new BytesArray(bytes));
+        }
     }
 
     public OpenSearchToken createNewApiToken(String tokenName) {
@@ -705,9 +711,9 @@ public ClusterInfo mainInfo() {
         }
         String versionNumber = versionBody.get("number");
         OpenSearchMajorVersion major = OpenSearchMajorVersion.parse(versionNumber);
-        if (major.before(OpenSearchMajorVersion.V_2_X)) {
+        if (major.before(OpenSearchMajorVersion.V_1_X)) {
             throw new OpenSearchHadoopIllegalStateException("Invalid major version [" + major + "]. " +
-                    "Version is lower than minimum required version [" + OpenSearchMajorVersion.V_2_X + "].");
+                    "Version is lower than minimum required version [" + OpenSearchMajorVersion.V_1_X + "].");
         }
         return new ClusterInfo(new ClusterName(clusterName, clusterUUID), OpenSearchMajorVersion.parse(versionNumber));
     }

mr/src/main/java/org/opensearch/hadoop/util/OpenSearchMajorVersion.java

@@ -39,8 +39,10 @@ public class OpenSearchMajorVersion implements Serializable {
 
     private static final long serialVersionUID = 1L;
 
+    public static final OpenSearchMajorVersion V_1_X = new OpenSearchMajorVersion((byte) 1, "1.x");
     public static final OpenSearchMajorVersion V_2_X = new OpenSearchMajorVersion((byte) 2, "2.x");
     public static final OpenSearchMajorVersion V_3_X = new OpenSearchMajorVersion((byte) 3, "3.x");
+    public static final OpenSearchMajorVersion V_7_X = new OpenSearchMajorVersion((byte) 7, "7.x");
     public static final OpenSearchMajorVersion LATEST = V_3_X;
 
     public final byte major;
@@ -88,6 +90,9 @@ public static OpenSearchMajorVersion parse(String version) {
         if (version.startsWith("3.")) {
             return new OpenSearchMajorVersion((byte) 3, version);
         }
+        if (version.startsWith("7.")) {
+            return new OpenSearchMajorVersion((byte) 7, version);
+        }
         throw new OpenSearchHadoopIllegalArgumentException("Unsupported/Unknown OpenSearch version [" + version + "]." +
                 "Highest supported version is [" + LATEST.version + "]. You may need to upgrade OpenSearch-Hadoop.");
     }

mr/src/test/java/org/opensearch/hadoop/rest/ApplyAliasMetaDataTest.java

@@ -54,6 +54,8 @@ public class ApplyAliasMetaDataTest {
 
     private static final OpenSearchMajorVersion[] OPENSEARCH_VERSIONS =
             new OpenSearchMajorVersion[]{
+                    OpenSearchMajorVersion.V_1_X,
+                    OpenSearchMajorVersion.V_2_X,
                     OpenSearchMajorVersion.V_3_X
             };
 

mr/src/test/java/org/opensearch/hadoop/rest/ResourceTest.java

@@ -58,7 +58,10 @@ public static Collection<Object[]> params() {
         parameters.add(new Object[]{OpenSearchMajorVersion.V_2_X, true});
 
         parameters.add(new Object[]{OpenSearchMajorVersion.V_3_X, false});
-        parameters.add(new Object[]{OpenSearchMajorVersion.V_2_X, false});
+        parameters.add(new Object[] { OpenSearchMajorVersion.V_2_X, false });
+
+        parameters.add(new Object[] { OpenSearchMajorVersion.V_1_X, true });
+        parameters.add(new Object[] { OpenSearchMajorVersion.V_1_X, false });
         return parameters;
     }
 

mr/src/test/java/org/opensearch/hadoop/rest/RestClientTest.java

@@ -275,31 +275,6 @@ public void testMainInfo() {
         assertNotNull(clusterInfo.getClusterName().getUUID());
     }
 
-    @Test
-    public void testMainInfoWithClusterTooOld() {
-        String response = "{\n" +
-                "\"name\": \"node\",\n" +
-                "\"cluster_name\": \"cluster\",\n" +
-                "\"version\": {\n" +
-                "  \"number\": \"1.0.0\"\n" +
-                "},\n" +
-                "\"tagline\": \"The OpenSearch Project: https://opensearch.org/\"\n" +
-                "}";
-
-        NetworkClient mock = Mockito.mock(NetworkClient.class);
-        Mockito.when(mock.execute(Mockito.any(SimpleRequest.class), Mockito.eq(true)))
-                .thenReturn(new SimpleResponse(201, new FastByteArrayInputStream(new BytesArray(response)), "localhost:9200"));
-
-        RestClient client = new RestClient(new TestSettings(), mock);
-
-        try {
-            client.mainInfo();
-            fail("Shouldn't operate on main version that is too old.");
-        } catch (OpenSearchHadoopIllegalStateException e) {
-            assertEquals("Invalid major version [1.0.0]. Version is lower than minimum required version [2.x].", e.getMessage());
-        }
-    }
-
     @Test
     public void testMainInfoWithClusterNotProvidingUUID() {
         String response = "{\n" +

mr/src/test/java/org/opensearch/hadoop/serialization/CommandTest.java

@@ -81,7 +81,8 @@ public static Collection<Object[]> data() {
                 ConfigurationOptions.OPENSEARCH_OPERATION_UPSERT,
                 ConfigurationOptions.OPENSEARCH_OPERATION_DELETE};
         boolean[] asJsons = new boolean[]{false, true};
-        OpenSearchMajorVersion[] versions = new OpenSearchMajorVersion[]{OpenSearchMajorVersion.V_2_X,
+        OpenSearchMajorVersion[] versions = new OpenSearchMajorVersion[]{
+                OpenSearchMajorVersion.V_1_X, OpenSearchMajorVersion.V_2_X,
                 OpenSearchMajorVersion.V_3_X};
 
         for (OpenSearchMajorVersion version : versions) {

mr/src/test/java/org/opensearch/hadoop/util/OpenSearchMajorVersionTest.java

@@ -66,6 +66,7 @@ public class OpenSearchMajorVersionTest {
         versions.add("1.3.5");
         versions.add("1.3.6");
         versions.add("1.3.7");
+        versions.add("1.3.8");
         versions.add("2.0.0-alpha1");
         versions.add("2.0.0-beta2");
         versions.add("2.0.0-rc3");
@@ -78,6 +79,8 @@ public class OpenSearchMajorVersionTest {
         versions.add("2.3.0");
         versions.add("2.3.1");
         versions.add("2.4.0");
+        versions.add("2.5.0");
+        versions.add("2.6.0");
         versions.add("3.0.0-alpha1");
         versions.add("3.0.0-beta2");
         versions.add("3.0.0-rc3");

spark/sql-20/src/itest/scala/org/opensearch/spark/sql/streaming/StreamingQueryTestHarness.scala

@@ -400,7 +400,7 @@ object TestingSerde extends Serializable {
   }
 
   def deserialize[T](line: String): T = {
-    val data: T = IOUtils.deserializeFromBase64(line, Dataset.class)
+    val data: T = IOUtils.deserializeFromBase64(line, T.class)
     data
   }
 }

spark/sql-30/src/itest/scala/org/opensearch/spark/sql/streaming/StreamingQueryTestHarness.scala

@@ -400,7 +400,7 @@ object TestingSerde extends Serializable {
   }
 
   def deserialize[T](line: String): T = {
-    val data: T = IOUtils.deserializeFromBase64(line, Dataset.class)
+    val data: T = IOUtils.deserializeFromBase64(line, T.class)
     data
   }
 }