Lots of changes to sync with Hadoop 2.2/Cascading 2.5

Not done yet, as there's a Kryo issue in cascading.utils that needs to be fixed upstream
ScaleUnlimited · Oct 25, 2014 · 09894a4 · 09894a4
1 parent 6a3dc35
commit 09894a4
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 18 deletions.
diff --git a/build.properties b/build.properties
@@ -36,7 +36,7 @@ build.dir.test-reports=${build.dir}/test
 javac.debug=on
 javac.optimize=on
 javac.deprecation=off
-javac.version=1.6
+javac.version=1.7
 javac.args=
 javac.args.warnings=-Xlint:none
 build.encoding=UTF-8
diff --git a/build.xml b/build.xml
@@ -226,7 +226,7 @@
 			</settings>
 			<project name="${ant.project.name}" />
 			<classpath>
-				<container path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6" />
+				<container path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7" />
 
 				<source path="${main.src.dir}"
 				        output="${build.dir.main-classes-eclipse}" />

diff --git a/pom.xml b/pom.xml
@@ -35,7 +35,7 @@
 
     <properties>
         <cascading.version>2.5.6</cascading.version>
-        <hadoop.version>2.5.1</hadoop.version>
+        <hadoop.version>2.2.0</hadoop.version>
         <solr.version>4.10.1</solr.version>
     </properties>
 
@@ -75,6 +75,13 @@
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-core</artifactId>
             <version>${solr.version}</version>
+            <exclusions>
+                <!-- Hadoop-annotations is pulling in JDK tools as a system scope dependency -->
+                <exclusion>
+                    <groupId>jdk.tools</groupId>
+                    <artifactId>jdk.tools</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -83,6 +90,18 @@
             <version>1.7.2</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.codehaus.jackson</groupId>
+            <artifactId>jackson-mapper-asl</artifactId>
+            <version>1.8.3</version>
+        </dependency>
+
         <!-- Test -->
 
 		<dependency>
@@ -92,7 +111,21 @@
             <scope>test</scope>
         </dependency>
 
-		<dependency>
+        <dependency>
+            <!-- Commons-logging used by Solr during testing needs commons logging -->
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>1.1.1</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-httpclient</groupId>
+            <artifactId>commons-httpclient</artifactId>
+            <version>3.1</version>
+         </dependency>
+
+        <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
             <version>1.9.0</version>
@@ -114,6 +147,7 @@
 
         <!-- Provided -->
 
+            <!-- 
 		<dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
@@ -136,8 +170,8 @@
                     <artifactId>ant</artifactId>
                 </exclusion>
             </exclusions>
-            <scope>provided</scope>
         </dependency>
+                     -->
 
 		<dependency>
             <groupId>ant-eclipse</groupId>

diff --git a/src/main/java/com/scaleunlimited/cascading/scheme/core/SolrSchemeUtil.java b/src/main/java/com/scaleunlimited/cascading/scheme/core/SolrSchemeUtil.java
@@ -61,16 +61,14 @@ public static void validate(File solrCoreDir, String dataDirPropertyName, Fields
         File tmpDataDir = new File(tmpFolder, UUID.randomUUID().toString());
         tmpDataDir.mkdir();
 
-        System.setProperty("solr.solr.home", tmpSolrHome.getAbsolutePath());
         System.setProperty(dataDirPropertyName, tmpDataDir.getAbsolutePath());
         System.setProperty("enable.special-handlers", "false"); // All we need is the update request handler
         System.setProperty("enable.cache-warming", "false"); // We certainly don't need to warm the cache
 
-        CoreContainer.Initializer initializer = new CoreContainer.Initializer();
-        CoreContainer coreContainer = null;
+        CoreContainer coreContainer = new CoreContainer(tmpSolrHome.getAbsolutePath());
 
         try {
-            coreContainer = initializer.initialize();
+            coreContainer.load();
             Collection<SolrCore> cores = coreContainer.getCores();
             SolrCore core = null;
 
@@ -82,7 +80,7 @@ public static void validate(File solrCoreDir, String dataDirPropertyName, Fields
                 core = cores.iterator().next();
             }
 
-            IndexSchema schema = core.getSchema();
+            IndexSchema schema = core.getLatestSchema();
             Map<String, SchemaField> solrFields = schema.getFields();
             Set<String> schemeFieldnames = new HashSet<String>();
 

diff --git a/src/main/java/com/scaleunlimited/cascading/scheme/core/SolrWriter.java b/src/main/java/com/scaleunlimited/cascading/scheme/core/SolrWriter.java
@@ -34,18 +34,18 @@ public SolrWriter(KeepAliveHook keepAlive, Fields sinkFields, String dataDirProp
         _maxSegments = maxSegments;
 
         _updateRequest = new BinaryUpdateRequest();
-        // Set up overwite=false. See https://issues.apache.org/jira/browse/SOLR-653
+        // Set up overwrite=false. See https://issues.apache.org/jira/browse/SOLR-653
         // for details why we have to do it this way.
         _updateRequest.setParam(UpdateParams.OVERWRITE, Boolean.toString(false));
 
         // Fire up an embedded Solr server
         try {
-            System.setProperty("solr.solr.home", SolrSchemeUtil.makeTempSolrHome(solrCoreDir).getAbsolutePath());
             System.setProperty(dataDirPropertyName, dataDir);
             System.setProperty("enable.special-handlers", "false"); // All we need is the update request handler
             System.setProperty("enable.cache-warming", "false"); // We certainly don't need to warm the cache
-            CoreContainer.Initializer initializer = new CoreContainer.Initializer();
-            _coreContainer = initializer.initialize();
+            File solrHome = SolrSchemeUtil.makeTempSolrHome(solrCoreDir);
+            _coreContainer = new CoreContainer(solrHome.getAbsolutePath());
+            _coreContainer.load();
             _solrServer = new EmbeddedSolrServer(_coreContainer, solrCoreDir.getName());
         } catch (Exception e) {
             if (_coreContainer != null) {

diff --git a/src/test/java/com/scaleunlimited/cascading/scheme/core/AbstractSolrSchemeTest.java b/src/test/java/com/scaleunlimited/cascading/scheme/core/AbstractSolrSchemeTest.java
@@ -146,11 +146,10 @@ protected void testSimpleIndexing() throws Exception {
         flow.complete();
 
         // Open up the Solr index, and do some searches.
-        System.setProperty("solr.solr.home", SOLR_HOME_DIR);
         System.setProperty("solr.data.dir", out + "/part-00000");
-        CoreContainer.Initializer initializer = new CoreContainer.Initializer();
-        CoreContainer coreContainer;
-        coreContainer = initializer.initialize();
+
+        CoreContainer coreContainer = new CoreContainer(SOLR_HOME_DIR);
+        coreContainer.load();
         SolrServer solrServer = new EmbeddedSolrServer(coreContainer, "");
 
         ModifiableSolrParams params = new ModifiableSolrParams();

diff --git a/src/test/resources/solr-home-4.1/collection1/conf/_rest_managed.json b/src/test/resources/solr-home-4.1/collection1/conf/_rest_managed.json
@@ -0,0 +1,3 @@
+{
+  "initArgs":{},
+  "managedList":[]}