[GEARPUMP-222] remove DFSJarStore from gearpump-daemon

Author: huafengw <fvunicorn@gmail.com> Closes #96 from huafengw/jarclass.
apache · Oct 12, 2016 · a01809b · a01809b
1 parent 1b98891
commit a01809b
Show file tree

Hide file tree

Showing 7 changed files with 45 additions and 18 deletions.
diff --git a/...ces/org.apache.gearpump.jarstore.JarStore → ...ces/org.apache.gearpump.jarstore.JarStore b/...ces/org.apache.gearpump.jarstore.JarStore → ...ces/org.apache.gearpump.jarstore.JarStore
@@ -16,5 +16,4 @@
 # limitations under the License.
 #
 
-org.apache.gearpump.jarstore.local.LocalJarStore
-org.apache.gearpump.jarstore.dfs.DFSJarStore
+org.apache.gearpump.jarstore.local.LocalJarStore
diff --git a/...ces/org.apache.gearpump.jarstore.JarStore → ...ces/org.apache.gearpump.jarstore.JarStore b/...ces/org.apache.gearpump.jarstore.JarStore → ...ces/org.apache.gearpump.jarstore.JarStore
@@ -16,5 +16,4 @@
 # limitations under the License.
 #
 
-org.apache.gearpump.jarstore.local.LocalJarStore
-org.apache.gearpump.jarstore.dfs.DFSJarStore
+org.apache.gearpump.jarstore.local.LocalJarStore
diff --git a/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java b/experiments/cgroup/src/main/java/org/apache/gearpump/cluster/utils/SystemOperation.java
@@ -17,11 +17,12 @@
  */
 package org.apache.gearpump.cluster.utils;
 
-import org.apache.commons.io.IOUtils;
+import org.apache.gearpump.google.common.io.CharStreams;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 public class SystemOperation {
 
@@ -44,8 +45,8 @@ public static String exec(String cmd) throws IOException {
     Process process = new ProcessBuilder(new String[]{"/bin/bash", "-c", cmd}).start();
     try {
       process.waitFor();
-      String output = IOUtils.toString(process.getInputStream());
-      String errorOutput = IOUtils.toString(process.getErrorStream());
+      String output = CharStreams.toString(new InputStreamReader(process.getInputStream()));
+      String errorOutput = CharStreams.toString(new InputStreamReader(process.getErrorStream()));
       LOG.debug("Shell Output: " + output);
       if (errorOutput.length() != 0) {
         LOG.error("Shell Error Output: " + errorOutput);

diff --git a/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore b/gearpump-hadoop/src/main/resources/META-INF/services/org.apache.gearpump.jarstore.JarStore
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gearpump.jarstore.dfs.DFSJarStore
diff --git a/...e/gearpump/jarstore/dfs/DFSJarStore.scala → ...e/gearpump/jarstore/dfs/DFSJarStore.scala b/...e/gearpump/jarstore/dfs/DFSJarStore.scala → ...e/gearpump/jarstore/dfs/DFSJarStore.scala
@@ -18,12 +18,11 @@
 package org.apache.gearpump.jarstore.dfs
 
 import java.io.{InputStream, OutputStream}
-
-import com.typesafe.config.Config
-import org.apache.gearpump.jarstore.JarStore
 import org.apache.gearpump.util.Constants
+import org.apache.gearpump.jarstore.JarStore
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
+import com.typesafe.config.Config
 import org.apache.hadoop.fs.permission.{FsAction, FsPermission}
 
 /**

diff --git a/project/Build.scala b/project/Build.scala
@@ -145,8 +145,7 @@ object Build extends sbt.Build {
       "com.typesafe.akka" %% "akka-cluster" % akkaVersion,
       "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion,
       "commons-logging" % "commons-logging" % commonsLoggingVersion,
-      "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion,
-      "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided"
+      "com.typesafe.akka" %% "akka-distributed-data-experimental" % akkaVersion
     )
   )
 
@@ -258,7 +257,7 @@ object Build extends sbt.Build {
     base = file("."),
     settings = commonSettings ++ noPublish ++ gearpumpUnidocSetting)
       .aggregate(shaded, core, daemon, streaming, services, external_kafka, external_monoid,
-      external_serializer, examples, storm, yarn, external_hbase, packProject,
+      external_serializer, examples, storm, yarn, external_hbase, gearpumpHadoop, packProject,
       external_hadoopfs, integration_test).settings(Defaults.itSettings: _*)
       .disablePlugins(sbtassembly.AssemblyPlugin)
 
@@ -463,14 +462,24 @@ object Build extends sbt.Build {
       .dependsOn (streaming % "test->test; compile->compile")
       .disablePlugins(sbtassembly.AssemblyPlugin)
 
+  lazy val gearpumpHadoop = Project(
+    id = "gearpump-hadoop",
+    base = file("gearpump-hadoop"),
+    settings = commonSettings ++ noPublish ++
+      Seq(
+        libraryDependencies ++= Seq(
+          "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion,
+          "org.apache.hadoop" % "hadoop-common" % hadoopVersion
+        )
+      )
+  ).dependsOn(core % "compile->compile").disablePlugins(sbtassembly.AssemblyPlugin)
+
   lazy val yarn = Project(
     id = "gearpump-experiments-yarn",
     base = file("experiments/yarn"),
     settings = commonSettings ++ noPublish ++
       Seq(
         libraryDependencies ++= Seq(
-          "org.apache.hadoop" % "hadoop-hdfs" % hadoopVersion,
-          "org.apache.hadoop" % "hadoop-common" % hadoopVersion,
           "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion,
           "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion,
           "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion,
@@ -480,8 +489,8 @@ object Build extends sbt.Build {
           "org.apache.hadoop" % "hadoop-yarn-server-nodemanager" % hadoopVersion % "provided"
         )
       ))
-      .dependsOn(services % "test->test;compile->compile", daemon % "provided", core % "provided")
-      .disablePlugins(sbtassembly.AssemblyPlugin)
+      .dependsOn(services % "test->test;compile->compile", daemon % "provided",
+        core % "provided", gearpumpHadoop).disablePlugins(sbtassembly.AssemblyPlugin)
 
   lazy val external_hbase = Project(
     id = "gearpump-external-hbase",

diff --git a/project/Pack.scala b/project/Pack.scala
@@ -114,7 +114,8 @@ object Pack extends sbt.Build {
         packLibDir := Map(
           "lib" -> new ProjectsToPack(core.id, streaming.id),
           "lib/daemon" -> new ProjectsToPack(daemon.id, cgroup.id).exclude(core.id, streaming.id),
-          "lib/yarn" -> new ProjectsToPack(yarn.id).exclude(services.id, daemon.id),
+          "lib/yarn" -> new ProjectsToPack(gearpumpHadoop.id, yarn.id).
+            exclude(services.id, daemon.id, core.id),
           "lib/services" -> new ProjectsToPack(services.id).exclude(daemon.id),
           "lib/storm" -> new ProjectsToPack(storm.id).exclude(streaming.id)
         ),