[Refactor] From Legacy to OpenSearch Namespace with Hive IT passing (#31

)

More namespace refactoring along with resource file directory renaming. Hive
integration tests passing.

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>

buildSrc/src/main/groovy/org/opensearch/hadoop/gradle/BuildPlugin.groovy

@@ -449,10 +449,10 @@ class BuildPlugin implements Plugin<Project>  {
         project.tasks.withType(Javadoc) { Javadoc javadoc ->
             javadoc.title = "${project.rootProject.description} ${project.version} API"
             javadoc.excludes = [
-                    "org/elasticsearch/hadoop/mr/compat/**",
-                    "org/elasticsearch/hadoop/rest/**",
-                    "org/elasticsearch/hadoop/serialization/**",
-                    "org/elasticsearch/hadoop/util/**",
+                    "org/opensearch/hadoop/mr/compat/**",
+                    "org/opensearch/hadoop/rest/**",
+                    "org/opensearch/hadoop/serialization/**",
+                    "org/opensearch/hadoop/util/**",
                     "org/apache/hadoop/hive/**"
             ]
             // Set javadoc executable to runtime Java (1.8)
@@ -467,11 +467,11 @@ class BuildPlugin implements Plugin<Project>  {
             javadocOptions.showFromProtected()
             javadocOptions.addStringOption('Xdoclint:none', '-quiet')
             javadocOptions.groups = [
-                    'Elasticsearch Map/Reduce' : ['org.elasticsearch.hadoop.mr*'],
-                    'Elasticsearch Hive' : ['org.elasticsearch.hadoop.hive*'],
-                    'Elasticsearch Pig' : ['org.elasticsearch.hadoop.pig*'],
-                    'Elasticsearch Spark' : ['org.elasticsearch.spark*'],
-                    'Elasticsearch Storm' : ['org.elasticsearch.storm*'],
+                    'OpenSearch Map/Reduce' : ['org.opensearch.hadoop.mr*'],
+                    'OpenSearch Hive' : ['org.opensearch.hadoop.hive*'],
+                    'OpenSearch Pig' : ['org.opensearch.hadoop.pig*'],
+                    'OpenSearch Spark' : ['org.opensearch.spark*'],
+                    'OpenSearch Storm' : ['org.opensearch.storm*'],
             ]
             javadocOptions.links = [ // External doc links
                     "https://docs.oracle.com/javase/8/docs/api/",

hive/src/itest/java/org/opensearch/hadoop/integration/hive/AbstractHiveExtraTests.java

@@ -97,7 +97,7 @@ public void testQuery() throws Exception {
     public void testDate() throws Exception {
         String resource = "hive-date-as-long";
         RestUtils.touch("hive-date-as-long");
-        RestUtils.putMapping("hive-date-as-long", "data", "org/elasticsearch/hadoop/hive/hive-date-typeless-mapping.json");
+        RestUtils.putMapping("hive-date-as-long", "data", "org/opensearch/hadoop/hive/hive-date-typeless-mapping.json");
 
         String docEndpoint = docEndpoint(resource, "data", TestUtils.getOpenSearchClusterInfo().getMajorVersion());
 

mr/src/itest/java/org/opensearch/hadoop/integration/mr/AbstractMROldApiSaveTest.java

@@ -539,7 +539,7 @@ public void testNested() throws Exception {
         conf.set(ConfigurationOptions.OPENSEARCH_RESOURCE, resource("mroldapi-nested", "data", clusterInfo.getMajorVersion()));
         conf.set(ConfigurationOptions.ES_INDEX_AUTO_CREATE, "no");
 
-        RestUtils.putMapping(indexPrefix + "mroldapi-nested", "data", "org/elasticsearch/hadoop/integration/mr-nested.json");
+        RestUtils.putMapping(indexPrefix + "mroldapi-nested", "data", "org/opensearch/hadoop/integration/mr-nested.json");
 
         runJob(conf);
     }

mr/src/main/java/org/opensearch/hadoop/cfg/Settings.java

@@ -62,7 +62,7 @@ public abstract class Settings {
     public OpenSearchMajorVersion getInternalVersionOrThrow() {
         String version = getProperty(InternalConfigurationOptions.INTERNAL_OPENSEARCH_VERSION, null);
         if (version == null) {
-            throw new IllegalArgumentException("Elasticsearch version:[ " + InternalConfigurationOptions.INTERNAL_OPENSEARCH_VERSION + "] not present in configuration");
+            throw new IllegalArgumentException("OpenSearch version:[ " + InternalConfigurationOptions.INTERNAL_OPENSEARCH_VERSION + "] not present in configuration");
         }
         return OpenSearchMajorVersion.parse(version);
     }
@@ -91,7 +91,7 @@ public OpenSearchMajorVersion getInternalVersionOrLatest() {
     public ClusterInfo getClusterInfoOrThrow() {
         ClusterInfo clusterInfo = getClusterInfoOrNull();
         if (clusterInfo == null) {
-            throw new IllegalArgumentException("Elasticsearch cluster name:[ " + InternalConfigurationOptions.INTERNAL_OPENSEARCH_CLUSTER_NAME +
+            throw new IllegalArgumentException("OpenSearch cluster name:[ " + InternalConfigurationOptions.INTERNAL_OPENSEARCH_CLUSTER_NAME +
                     "] not present in configuration");
         }
         return clusterInfo;

mr/src/main/java/org/opensearch/hadoop/mr/OpenSearchMapReduceUtil.java

@@ -83,14 +83,14 @@ public static void initCredentials(Job job) {
                 TokenUtil.addTokenForJob(bootstrap, clusterInfo.getClusterName(), user, job);
             } catch (OpenSearchHadoopException ex) {
                 throw new OpenSearchHadoopIllegalArgumentException(String.format("Cannot detect OpenSearch version - "
-                        + "typically this happens if the network/Elasticsearch cluster is not accessible or when targeting "
+                        + "typically this happens if the network/OpenSearch cluster is not accessible or when targeting "
                         + "a WAN/Cloud instance without the proper setting '%s'", ConfigurationOptions.OPENSEARCH_NODES_WAN_ONLY), ex);
             } finally {
                 bootstrap.close();
             }
         } else {
             if (LOG.isDebugEnabled()) {
-                LOG.debug("Ignoring Elasticsearch credentials since Kerberos Auth is not enabled.");
+                LOG.debug("Ignoring OpenSearch credentials since Kerberos Auth is not enabled.");
             }
         }
     }
@@ -118,14 +118,14 @@ public static void initCredentials(JobConf jobConf) {
                 TokenUtil.addTokenForJobConf(bootstrap, clusterInfo.getClusterName(), user, jobConf);
             } catch (OpenSearchHadoopException ex) {
                 throw new OpenSearchHadoopIllegalArgumentException(String.format("Cannot detect OpenSearch version - "
-                        + "typically this happens if the network/Elasticsearch cluster is not accessible or when targeting "
+                        + "typically this happens if the network/OpenSearch cluster is not accessible or when targeting "
                         + "a WAN/Cloud instance without the proper setting '%s'", ConfigurationOptions.OPENSEARCH_NODES_WAN_ONLY), ex);
             } finally {
                 bootstrap.close();
             }
         } else {
             if (LOG.isDebugEnabled()) {
-                LOG.debug("Ignoring Elasticsearch credentials since Kerberos Auth is not enabled.");
+                LOG.debug("Ignoring OpenSearch credentials since Kerberos Auth is not enabled.");
             }
         }
     }

mr/src/main/java/org/opensearch/hadoop/serialization/bulk/PerEntityPoolingMetadataExtractor.java

@@ -89,7 +89,7 @@ public UnsupportedMetadataFieldExtractor(Metadata unsupportedMetadata, OpenSearc
         @Override
         public Object field(Object target) {
             throw new OpenSearchHadoopUnsupportedOperationException("Unsupported metadata tag [" + unsupportedMetadata.getName()
-                    + "] for Elasticsearch version [" + version.toString() + "]. Bailing out...");
+                    + "] for OpenSearch version [" + version.toString() + "]. Bailing out...");
         }
     }
 

mr/src/main/java/org/opensearch/hadoop/serialization/dto/mapping/MappingUtils.java

@@ -73,7 +73,7 @@ public static void validateMapping(Collection<String> fields, Mapping mapping, F
             return;
         }
 
-        String message = String.format("Field(s) [%s] not found in the Elasticsearch mapping specified; did you mean [%s]?",
+        String message = String.format("Field(s) [%s] not found in the OpenSearch mapping specified; did you mean [%s]?",
                 removeDoubleBrackets(results[0]), removeDoubleBrackets(results[1]));
         if (validation == FieldPresenceValidation.WARN) {
             log.warn(message);

mr/src/main/java/org/opensearch/hadoop/serialization/field/ConstantFieldExtractor.java

@@ -36,7 +36,7 @@
 
 public class ConstantFieldExtractor implements FieldExtractor, SettingsAware {
 
-    public static final String PROPERTY = "org.elasticsearch.hadoop.serialization.ConstantFieldExtractor.property";
+    public static final String PROPERTY = "org.opensearch.hadoop.serialization.ConstantFieldExtractor.property";
     private List<String> fieldNames;
     private Object value;
     private boolean autoQuote = true;

mr/src/main/java/org/opensearch/hadoop/util/Version.java

@@ -140,13 +140,13 @@ public static String versionHashShort() {
     public static void logVersion() {
         if (!printed) {
             printed = true;
-            LogFactory.getLog(Version.class).info("Elasticsearch Hadoop " + version());
+            LogFactory.getLog(Version.class).info("OpenSearch Hadoop " + version());
 
             // Check java version
             String javaVersion = System.getProperty("java.version");
             if (javaVersion.startsWith("1.")) {
                 if (!javaVersion.startsWith("1.8.")) {
-                    LogFactory.getLog(Version.class).warn("Using java version " + javaVersion + " is deprecated in Elasticsearch Hadoop");
+                    LogFactory.getLog(Version.class).warn("Using java version " + javaVersion + " is deprecated in OpenSearch Hadoop");
                 }
             }
         }

mr/src/test/java/org/opensearch/hadoop/rest/PartitionDefinitionTest.java

@@ -53,7 +53,7 @@ public class PartitionDefinitionTest {
     private Mapping getTestMapping() throws IOException {
         ObjectMapper mapper = new ObjectMapper();
         JsonParser jsonParser = mapper.getJsonFactory()
-                .createJsonParser(getClass().getResourceAsStream("/org/elasticsearch/hadoop/serialization/dto/mapping/typeless/basic.json"));
+                .createJsonParser(getClass().getResourceAsStream("/org/opensearch/hadoop/serialization/dto/mapping/typeless/basic.json"));
         Map<String, Object> map =
                 (Map<String, Object>) mapper.readValue(jsonParser, Map.class);
         return FieldParser.parseTypelessMappings(map).getResolvedView();

mr/src/test/java/org/opensearch/hadoop/rest/bulk/BulkProcessorTest.java

@@ -98,7 +98,7 @@ public BulkProcessorTest(OpenSearchMajorVersion version, org.opensearch.hadoop.r
 
     @Before
     public void setUp() throws Exception {
-        inputEntry = IOUtils.asString(getClass().getResourceAsStream("/org/elasticsearch/hadoop/rest/bulk-retry-input-template.json"));
+        inputEntry = IOUtils.asString(getClass().getResourceAsStream("/org/opensearch/hadoop/rest/bulk-retry-input-template.json"));
 
         testSettings = new TestSettings();
         testSettings.setResourceWrite("foo/bar");

mr/src/test/java/org/opensearch/hadoop/util/IOUtilsTest.java

@@ -54,7 +54,7 @@
 public class IOUtilsTest {
     @Test
     public void openResource() throws Exception {
-        InputStream inputStream = IOUtils.open("org/elasticsearch/hadoop/util/textdata.txt");
+        InputStream inputStream = IOUtils.open("org/opensearch/hadoop/util/textdata.txt");
         assertNotNull(inputStream);
 
         BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
@@ -83,24 +83,24 @@ public void openNonExistingFile() throws Exception {
 
     @Test
     public void testToCanonicalFile() throws Exception {
-        String file = "file:/some/path/org/elasticsearch/hadoop/util/Version.class";
+        String file = "file:/some/path/org/opensearch/hadoop/util/Version.class";
         URL url = new URL(file);
         String canonicalFilePath = IOUtils.toCanonicalFilePath(url);
         assertEquals(file, canonicalFilePath);
 
-        url = new URL("jar:file:/some/path/opensearch-hadoop-7.17.0.jar!/org/elasticsearch/hadoop/util/Version.class");
+        url = new URL("jar:file:/some/path/opensearch-hadoop-7.17.0.jar!/org/opensearch/hadoop/util/Version.class");
         canonicalFilePath = IOUtils.toCanonicalFilePath(url);
         assertEquals("file:/some/path/opensearch-hadoop-7.17.0.jar", canonicalFilePath);
 
-        url = new URL("file:/some/path/../path/org/elasticsearch/hadoop/util/Version.class");
+        url = new URL("file:/some/path/../path/org/opensearch/hadoop/util/Version.class");
         canonicalFilePath = IOUtils.toCanonicalFilePath(url);
-        assertEquals("file:/some/path/org/elasticsearch/hadoop/util/Version.class", canonicalFilePath);
+        assertEquals("file:/some/path/org/opensearch/hadoop/util/Version.class", canonicalFilePath);
     }
 
     @Test
     public void testToCanonicalFileSpringBoot() throws Exception {
         String jarWithinJarPath = "file:/some/path/outer.jar!/BOOT-INF/lib/opensearch-hadoop-7.17.0.jar";
-        String file = jarWithinJarPath + "!/org/elasticsearch/hadoop/util/Version.class";
+        String file = jarWithinJarPath + "!/org/opensearch/hadoop/util/Version.class";
         URL url = new URL("jar", "", -1, file, new SpringBootURLStreamHandler(jarWithinJarPath) );
         String canonicalFilePath = IOUtils.toCanonicalFilePath(url);
         assertEquals("jar:" + jarWithinJarPath, canonicalFilePath);

pig/src/test/java/org/opensearch/hadoop/pig/PigJsonTest.java

@@ -36,6 +36,6 @@ public class PigJsonTest {
 
     @Test
     public void testJsonWithArray() throws Exception {
-        Map<String, Object> jsonMap = JsonUtils.mapFromJson("/org/elasticsearch/hadoop/pig/array.json");
+        Map<String, Object> jsonMap = JsonUtils.mapFromJson("/org/opensearch/hadoop/pig/array.json");
     }
 }

qa/kerberos/build.gradle

@@ -304,7 +304,7 @@ if (disableTests) {
                 f.addSetting("hadoop.proxyuser.hive.groups", "*")
 
                 // Add OpenSearch Security settings here because without them Spark will not obtain tokens
-                f.addSetting('es.security.authentication', 'kerberos')
+                f.addSetting('opensearch.security.authentication', 'kerberos')
                 f.addSetting('opensearch.net.spnego.auth.opensearch.principal', "${opensearchPrincipal}${realm}")
             }
             // SSL Server Config
@@ -468,7 +468,7 @@ if (disableTests) {
         jobSettings([
                 'opensearch.resource': 'qa_kerberos_mr_data',
                 'opensearch.nodes': opensearchAddress,
-                'es.security.authentication': 'kerberos',
+                'opensearch.security.authentication': 'kerberos',
                 'opensearch.net.spnego.auth.opensearch.principal': "${opensearchPrincipal}${realm}",
                 'load.field.names': 'number,name,url,picture,@timestamp,tag',
                 'mapreduce.map.java.opts': "-Xmx200m -Djava.security.krb5.conf=${krb5Conf.toString()}",
@@ -495,7 +495,7 @@ if (disableTests) {
         jobSettings([
                 'opensearch.resource': 'qa_kerberos_mr_data',
                 'es.nodes': opensearchAddress,
-                'es.security.authentication': 'kerberos',
+                'opensearch.security.authentication': 'kerberos',
                 'opensearch.net.spnego.auth.opensearch.principal': "${opensearchPrincipal}${realm}",
                 'mapreduce.map.java.opts': "-Xmx200m -Djava.security.krb5.conf=${krb5Conf.toString()}",
                 'mapreduce.reduce.java.opts': "-Xmx200m -Djava.security.krb5.conf=${krb5Conf.toString()}",
@@ -528,7 +528,7 @@ if (disableTests) {
         jobSettings([
                 'spark.opensearch.resource': 'qa_kerberos_spark_data',
                 'spark.opensearch.nodes': opensearchAddress,
-                'spark.es.security.authentication': 'kerberos',
+                'spark.opensearch.security.authentication': 'kerberos',
                 'spark.opensearch.net.spnego.auth.opensearch.principal': "${opensearchPrincipal}${realm}",
                 'spark.load.field.names': 'number,name,url,picture,@timestamp,tag',
                 'spark.yarn.am.memory': "200m",
@@ -560,7 +560,7 @@ if (disableTests) {
         jobSettings([
                 'spark.opensearch.resource': 'qa_kerberos_spark_data',
                 'spark.opensearch.nodes': opensearchAddress,
-                'spark.es.security.authentication': 'kerberos',
+                'spark.opensearch.security.authentication': 'kerberos',
                 'spark.opensearch.net.spnego.auth.opensearch.principal': "${opensearchPrincipal}${realm}",
                 'spark.yarn.am.memory': "200m",
                 'spark.driver.memory': "600m",

qa/kerberos/src/main/resources/hive/load_to_opensearch.sql

@@ -26,8 +26,8 @@ CREATE EXTERNAL TABLE IF NOT EXISTS es_artist_data (
 STORED BY 'org.opensearch.hadoop.hive.OpenSearchStorageHandler'
 TBLPROPERTIES(
   'opensearch.resource' = 'qa_kerberos_hive_data',
-  'es.security.authentication' = 'kerberos',
-  'es.net.spnego.auth.elasticsearch.principal' = 'HTTP/build.ci.opensearch.org@BUILD.CI.OPENSEARCH.ORG'
+  'opensearch.security.authentication' = 'kerberos',
+  'opensearch.net.spnego.auth.opensearch.principal' = 'HTTP/build.ci.opensearch.org@BUILD.CI.OPENSEARCH.ORG'
 );
 
 -- Create random timestamps up front since Hive's timestamp format differs from ISO8601

qa/kerberos/src/main/resources/pig/load_to_opensearch.pig

@@ -1,6 +1,6 @@
 A = LOAD '/data/artists' USING PigStorage('\t') AS (number: chararray, name: chararray, uri: chararray, picture: chararray, timestamp: chararray, tag: chararray);
 
 STORE A INTO 'qa_kerberos_pig_data' USING org.opensearch.pig.hadoop.OpenSearchStorage(
-    'es.security.authentication = kerberos',
+    'opensearch.security.authentication = kerberos',
     'opensearch.net.spnego.auth.opensearch.principal = HTTP/build.ci.opensearch.org@BUILD.CI.OPENSEARCH.ORG'
 );

qa/kerberos/src/main/resources/pig/read_from_opensearch.pig

@@ -1,5 +1,5 @@
 A = LOAD 'qa_kerberos_pig_data' USING org.opensearch.pig.hadoop.OpenSearchStorage(
-    'es.security.authentication = kerberos',
+    'opensearch.security.authentication = kerberos',
     'opensearch.net.spnego.auth.opensearch.principal = HTTP/build.ci.opensearch.org@BUILD.CI.OPENSEARCH.ORG'
 );
 

qa/kerberos/src/main/scala/org/elasticsearch/hadoop/qa/kerberos/spark/LoadToES.scala → qa/kerberos/src/main/scala/org/opensearch/hadoop/qa/kerberos/spark/LoadToES.scala

@@ -7,7 +7,7 @@
  * Modifications Copyright OpenSearch Contributors. See
  * GitHub history for details.
  */
- 
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -27,7 +27,7 @@
  * under the License.
  */
 
-package org.elasticsearch.hadoop.qa.kerberos.spark
+package org.opensearch.hadoop.qa.kerberos.spark
 
 import java.security.PrivilegedExceptionAction
 import org.apache.spark.SparkConf

qa/kerberos/src/main/scala/org/elasticsearch/hadoop/qa/kerberos/spark/ReadFromES.scala → qa/kerberos/src/main/scala/org/opensearch/hadoop/qa/kerberos/spark/ReadFromES.scala

@@ -7,7 +7,7 @@
  * Modifications Copyright OpenSearch Contributors. See
  * GitHub history for details.
  */
- 
+
 /*
  * Licensed to Elasticsearch under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -27,7 +27,7 @@
  * under the License.
  */
 
-package org.elasticsearch.hadoop.qa.kerberos.spark
+package org.opensearch.hadoop.qa.kerberos.spark
 
 import java.security.PrivilegedExceptionAction
 

storm/src/main/java/org/opensearch/storm/security/AutoOpenSearch.java

@@ -118,19 +118,19 @@ public void populateCredentials(Map<String, String> credentials, Map<String, Obj
         final Settings topologyAndClusterSettings = new CompositeSettings(Arrays.asList(topologyConf, clusterSettings));
 
         if (!AuthenticationMethod.KERBEROS.equals(topologyAndClusterSettings.getSecurityAuthenticationMethod())) {
-            throw new OpenSearchHadoopIllegalArgumentException("Configured Elasticsearch autocredential plugin but did not enable ES Kerberos [" +
+            throw new OpenSearchHadoopIllegalArgumentException("Configured OpenSearch autocredential plugin but did not enable OpenSearch Kerberos [" +
                     ConfigurationOptions.OPENSEARCH_SECURITY_AUTHENTICATION + "]. Bailing out...");
         }
 
         String userPrincipal = topologyAndClusterSettings.getProperty(USER_PRINCIPAL);
         if (userPrincipal == null) {
-            throw new OpenSearchHadoopIllegalArgumentException("Configured Elasticsearch autocredential plugin but did not provide [" +
+            throw new OpenSearchHadoopIllegalArgumentException("Configured OpenSearch autocredential plugin but did not provide [" +
                     USER_PRINCIPAL + "] setting. Bailing out...");
         }
 
         String userKeytab = topologyAndClusterSettings.getProperty(USER_KEYTAB);
         if (userKeytab == null) {
-            throw new OpenSearchHadoopIllegalArgumentException("Configured Elasticsearch autocredential plugin but did not provide [" +
+            throw new OpenSearchHadoopIllegalArgumentException("Configured OpenSearch autocredential plugin but did not provide [" +
                     USER_KEYTAB + "] setting. Bailing out...");
         }
 

test/shared/src/main/java/org/opensearch/hadoop/QueryTestParams.java

@@ -45,8 +45,8 @@
 
 public class QueryTestParams {
 
-    private static final String QUERY_DSL = "/org/elasticsearch/hadoop/integration/query.dsl";
-    private static final String QUERY_URI = "/org/elasticsearch/hadoop/integration/query.uri";
+    private static final String QUERY_DSL = "/org/opensearch/hadoop/integration/query.dsl";
+    private static final String QUERY_URI = "/org/opensearch/hadoop/integration/query.uri";
 
     private final File stagingLocation;
     private final boolean isLocal;