apache · warperwolf · Jul 21, 2020 · Mar 19, 2021 · Jul 27, 2020 · Jun 22, 2021
diff --git a/gradle/documentation/render-javadoc.gradle b/gradle/documentation/render-javadoc.gradle
@@ -151,8 +151,25 @@ configure(project(":solr:test-framework")) {
         "org.apache.solr.analysis",
         "org.apache.solr.cloud",
         "org.apache.solr.core",
+        "org.apache.solr.handler",
         "org.apache.solr.handler.component",
         "org.apache.solr.update.processor",
+        "org.apache.solr.util",
+        "org.apache.solr.search.similarities",
+        "org.apache.solr.search.function",
+        "org.apache.solr.search.facet",
+        "org.apache.solr.schema"
+    ]
+  }
+}
+
+configure(project(":solr:modules:hdfs")) {
+  project.tasks.withType(RenderJavadocTask) {
+    // TODO: clean up split packages
+    javadocMissingIgnore = [
+        "org.apache.solr.core",
+        "org.apache.solr.core.backup.repository",
+        "org.apache.solr.update",
         "org.apache.solr.util"
     ]
   }

diff --git a/gradle/validation/forbidden-apis.gradle b/gradle/validation/forbidden-apis.gradle
@@ -130,6 +130,11 @@ allprojects { prj ->
       forbiddenApisMain.exclude("**/jmh_generated/**")
     }
 
+    // Disable checks on code copied from Hadoop
+    if (prj.path == ":solr:modules:hdfs") {
+      forbiddenApisTest.exclude("**/org/apache/hadoop/**")
+    }
+
     // We rely on resolved configurations to compute the relevant set of rule
     // files for forbiddenApis. Since we don't want to resolve these configurations until
     // the task is executed, we can't really use them as task inputs properly. This is a
@@ -141,4 +146,4 @@ allprojects { prj ->
       task.inputs.dir(file(resources))
     }
   })
-}
+}
diff --git a/gradle/validation/rat-sources.gradle b/gradle/validation/rat-sources.gradle
@@ -102,6 +102,12 @@ allprojects {
                     exclude "src/test-files/META-INF/services/*"
                     break
 
+                case ":solr:modules:hdfs":
+                    exclude "src/test-files/**/*.aff"
+                    exclude "src/test-files/**/*.dic"
+                    exclude "src/test-files/**/*.incl"
+                    break
+
                 case ":solr:modules:langid":
                     exclude "**/langdetect-profiles/*"
                     break

diff --git a/gradle/validation/spotless.gradle b/gradle/validation/spotless.gradle
@@ -49,6 +49,7 @@ configure(project(":solr").subprojects) { prj ->
           case ":solr:modules:clustering":
           case ":solr:modules:extraction":
           case ":solr:modules:gcs-repository":
+          case ":solr:modules:hdfs":
           case ":solr:modules:langid":
           case ":solr:modules:ltr":
           case ":solr:modules:scripting":

diff --git a/gradle/validation/validate-source-patterns.gradle b/gradle/validation/validate-source-patterns.gradle
@@ -108,7 +108,6 @@ allprojects {
 configure(project(':solr:core')) {
   project.tasks.withType(ValidateSourcePatternsTask) {
     sourceFiles.exclude 'src/**/CheckLoggingConfiguration.java'
-    sourceFiles.exclude 'src/test/org/apache/hadoop/**'
   }
 }
 

diff --git a/settings.gradle b/settings.gradle
@@ -39,6 +39,7 @@ include "solr:modules:s3-repository"
 include "solr:modules:scripting"
 include "solr:modules:ltr"
 include "solr:modules:gcs-repository"
+include "solr:modules:hdfs"
 include "solr:webapp"
 include "solr:benchmark"
 include "solr:test-framework"

diff --git a/solr/core/build.gradle b/solr/core/build.gradle
@@ -66,7 +66,6 @@ dependencies {
   implementation 'commons-io:commons-io'
   implementation 'com.carrotsearch:hppc'
   implementation 'org.apache.commons:commons-collections4'
-  runtimeOnly 'commons-collections:commons-collections' // for Hadoop and...?
 
   implementation('com.github.ben-manes.caffeine:caffeine') { transitive = false }
 
@@ -136,27 +135,17 @@ dependencies {
   implementation ('org.apache.calcite:calcite-linq4j') { transitive = false }
   implementation ('org.apache.calcite.avatica:avatica-core') { transitive = false }
 
-  // Hadoop; general stuff
-  implementation ('org.apache.hadoop:hadoop-common') { transitive = false } // too many to ignore
-  implementation ('org.apache.hadoop:hadoop-annotations')
-  runtimeOnly 'org.apache.htrace:htrace-core4' // note: removed in Hadoop 3.3.2
-  runtimeOnly "org.apache.commons:commons-configuration2"
-  testImplementation ('org.apache.hadoop:hadoop-common::tests') { transitive = false }
-  // Hadoop Test hacks for Java Security Manager -- SOLR-14033
-  testImplementation 'org.apache.commons:commons-compress'
-  testImplementation ('com.sun.jersey:jersey-servlet') { transitive = false }
-
-  // HDFS
-  implementation ('org.apache.hadoop:hadoop-hdfs-client') { transitive = false }
-  testImplementation ('org.apache.hadoop:hadoop-hdfs') { transitive = false }
-  testImplementation ('org.apache.hadoop:hadoop-hdfs::tests') { transitive = false }
-  testImplementation 'org.apache.logging.log4j:log4j-1.2-api'
-
   // Hadoop auth framework
+  implementation ('org.apache.hadoop:hadoop-annotations')
   implementation ('org.apache.hadoop:hadoop-auth') { transitive = false }
-  runtimeOnly 'com.google.re2j:re2j' // transitive of hadoop-common; used by Kerberos auth
-  runtimeOnly ('org.apache.kerby:kerb-core')
-  runtimeOnly ('org.apache.kerby:kerb-util')
+  implementation ('org.apache.hadoop:hadoop-common') { transitive = false }
+  // transitive of hadoop-common; used by Kerberos auth
+  runtimeOnly 'commons-collections:commons-collections'
+  runtimeOnly 'com.google.re2j:re2j'
+  runtimeOnly 'org.apache.commons:commons-configuration2'
+  runtimeOnly 'org.apache.htrace:htrace-core4' // note: removed in Hadoop 3.3.2
+  runtimeOnly 'org.apache.kerby:kerb-core'
+  runtimeOnly 'org.apache.kerby:kerb-util'
 
   // Hadoop MiniKdc Dependencies (for Kerberos auth tests)
   testImplementation ('org.apache.hadoop:hadoop-minikdc', {

diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -36,6 +36,7 @@
 import org.apache.lucene.store.LockFactory;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.core.CachingDirectoryFactory.CloseListener;
+import org.apache.solr.update.UpdateLog;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -67,7 +68,6 @@ public enum DirContext {DEFAULT, META_DATA}
   public final static String LOCK_TYPE_NATIVE = "native";
   public final static String LOCK_TYPE_SINGLE = "single";
   public final static String LOCK_TYPE_NONE   = "none";
-  public final static String LOCK_TYPE_HDFS   = "hdfs";
 
   protected volatile CoreContainer coreContainer;
 
@@ -430,4 +430,12 @@ static DirectoryFactory loadDirectoryFactory(SolrConfig config, CoreContainer cc
     }
     return dirFactory;
   }
+
+  /**
+   * Returns a default Update Log instance. UpdateHandler will invoke this method if the
+   * solrconfig / plugin info does not specify any ulog class.
+   */
+  public UpdateLog newDefaultUpdateLog() {
+    return new UpdateLog();
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateHandler.java b/solr/core/src/java/org/apache/solr/update/UpdateHandler.java
@@ -115,12 +115,12 @@ public UpdateHandler(SolrCore core, UpdateLog updateLog)  {
     boolean skipUpdateLog = core.getCoreDescriptor().getCloudDescriptor() != null && !core.getCoreDescriptor().getCloudDescriptor().requiresTransactionLog();
     if (updateLog == null && ulogPluginInfo != null && ulogPluginInfo.isEnabled() && !skipUpdateLog) {
       DirectoryFactory dirFactory = core.getDirectoryFactory();
-      if (dirFactory instanceof HdfsDirectoryFactory) {
-        ulog = new HdfsUpdateLog(((HdfsDirectoryFactory)dirFactory).getConfDir());
-      } else {
-        ulog = ulogPluginInfo.className == null ? new UpdateLog():
-                core.getResourceLoader().newInstance(ulogPluginInfo, UpdateLog.class, true);
-      }
+
+      // if the update log class is not defined in the plugin info / solrconfig.xml
+      // (like <updateLog class="${solr.ulog:solr.UpdateLog}"> )
+      // we fall back use the one which is the default for the given directory factory
+      ulog = ulogPluginInfo.className == null ? dirFactory.newDefaultUpdateLog() :
+          core.getResourceLoader().newInstance(ulogPluginInfo, UpdateLog.class, true);
 
       if (!core.isReloaded() && !dirFactory.isPersistent()) {
         ulog.clearLog(core, ulogPluginInfo);

diff --git a/solr/core/src/test-files/core-site.xml b/solr/core/src/test-files/core-site.xml
@@ -18,6 +18,6 @@
 <configuration>
   <property>
     <name>hadoop.security.group.mapping</name>
-    <value>org.apache.solr.cloud.hdfs.FakeGroupMapping</value>
+    <value>org.apache.solr.security.hadoop.HadoopAuthFakeGroupMapping</value>
   </property>
 </configuration>
diff --git a/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java b/solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
@@ -76,7 +76,7 @@ public static void beforeTest() throws Exception {
 
     Properties nodeProperties = new Properties();
     // this sets the property for jetty starting SolrDispatchFilter
-    if (System.getProperty("solr.data.dir") == null && System.getProperty("solr.hdfs.home") == null) {
+    if (System.getProperty("solr.data.dir") == null) {
       nodeProperties.setProperty("solr.data.dir", createTempDir().toFile().getCanonicalPath());
     }
     jetty = new JettySolrRunner(homeDir.getAbsolutePath(), nodeProperties, buildJettyConfig("/solr"));