Skip to content
Browse files

Fixed bugs on running with Hadoop 1.0.x (#205).

  • Loading branch information...
1 parent 31420c3 commit 3b6041faed6830754e0a07816e41bedb3b5c8edf @akirakw akirakw committed Jul 5, 2012
View
38 ...ect/asakusa-jobqueue-server/src/main/dist/jobqueue-hadoop/libexec/configure-hadoop-cmd.sh
@@ -0,0 +1,38 @@
+#
+# Copyright 2011-2012 Asakusa Framework Team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ "$HADOOP_CMD" = "" ]
+then
+ if [ "$HADOOP_HOME" != "" ]
+ then
+ HADOOP_CMD="$HADOOP_HOME/bin/hadoop"
+ unset HADOOP_HOME
+ else
+ HADOOP_CMD="$(which hadoop)"
+ _RET=$?
+ if [ $_RET -ne 0 ]
+ then
+ echo 'hadoop command is not found' 1>&2
+ exit 1
+ fi
+ fi
+fi
+
+if [ ! -x "$HADOOP_CMD" ]
+then
+ echo "$HADOOP_CMD is not executable" 1>&2
+ exit 1
+fi
View
41 ...roject/asakusa-jobqueue-server/src/main/dist/jobqueue-hadoop/libexec/configure-libjars.sh
@@ -0,0 +1,41 @@
+#
+# Copyright 2011-2012 Asakusa Framework Team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -d "$ASAKUSA_HOME/core/lib" ]
+then
+ for f in $(ls "$ASAKUSA_HOME/core/lib")
+ do
+ if [ "$_JQ_LIBJARS" = "" ]
+ then
+ _JQ_LIBJARS="$ASAKUSA_HOME/core/lib/$f"
+ else
+ _JQ_LIBJARS="$_JQ_LIBJARS,$ASAKUSA_HOME/core/lib/$f"
+ fi
+ done
+fi
+
+if [ -d "$ASAKUSA_HOME/ext/lib" ]
+then
+ for f in $(ls "$ASAKUSA_HOME/ext/lib")
+ do
+ if [ "$_JQ_LIBJARS" = "" ]
+ then
+ _JQ_LIBJARS="$ASAKUSA_HOME/ext/lib/$f"
+ else
+ _JQ_LIBJARS="$_JQ_LIBJARS,$ASAKUSA_HOME/ext/lib/$f"
+ fi
+ done
+fi
View
66 ...e-project/asakusa-jobqueue-server/src/main/dist/jobqueue-hadoop/libexec/hadoop-execute.sh
@@ -44,19 +44,24 @@ Parameters:
EOF
}
+import() {
+ _SCRIPT="$1"
+ if [ -e "$_SCRIPT" ]
+ then
+ . "$_SCRIPT"
+ else
+ echo "$_SCRIPT is not found" 1>&2
+ exit 1
+ fi
+}
+
if [ $# -lt 6 ]
then
echo "Invalid arguments: [$@]" 1>&2
usage
exit 1
fi
-if [ "$ASAKUSA_HOME" = "" ]
-then
- echo '$ASAKUSA_HOME'" is not defined" 1>&2
- exit 1
-fi
-
_OPT_CLASS_NAME="$1"
shift
_OPT_BATCH_ID="$1"
@@ -71,39 +76,21 @@ _OPT_BATCH_ARGUMENTS="$1"
shift
_JQ_ROOT="$(dirname $0)/.."
-if [ -e "$_JQ_ROOT/conf/env.sh" ]
-then
- . "$_JQ_ROOT/conf/env.sh"
-fi
+import "$_JQ_ROOT/conf/env.sh"
+import "$_JQ_ROOT/libexec/validate-env.sh"
# Move to home directory
cd ~
_JQ_TOOL_LAUNCHER="com.asakusafw.runtime.stage.ToolLauncher"
-_JQ_CORE_LIB_DIR="$ASAKUSA_HOME/core/lib"
-_JQ_EXT_LIB_DIR="$ASAKUSA_HOME/ext/lib"
-_JQ_RUNTIME_LIB="$_JQ_CORE_LIB_DIR/asakusa-runtime-all.jar"
+_JQ_RUNTIME_LIB="$ASAKUSA_HOME/core/lib/asakusa-runtime-all.jar"
_JQ_PLUGIN_CONF="$ASAKUSA_HOME/core/conf/asakusa-resources.xml"
_JQ_APP_LIB="$ASAKUSA_HOME/batchapps/$_OPT_BATCH_ID/lib/jobflow-${_OPT_FLOW_ID}.jar"
-_JQ_LIBJAR="$_JQ_APP_LIB"
-
-if [ -d "$_JQ_CORE_LIB_DIR" ]
-then
- for f in $(ls "$_JQ_CORE_LIB_DIR")
- do
- _JQ_LIBJAR="$_JQ_LIBJAR,$_JQ_CORE_LIB_DIR/$f"
- done
-fi
-
-if [ -d "$_JQ_EXT_LIB_DIR" ]
-then
- for f in $(ls "$_JQ_EXT_LIB_DIR")
- do
- _JQ_LIBJAR="$_JQ_LIBJAR,$_JQ_EXT_LIB_DIR/$f"
- done
-fi
+_JQ_LIBJARS="$_JQ_APP_LIB"
+import "$_JQ_ROOT/libexec/configure-libjars.sh"
+import "$_JQ_ROOT/libexec/configure-hadoop-cmd.sh"
if [ "$HADOOP_TMP_DIR" != "" ]
then
@@ -115,25 +102,26 @@ then
fi
echo "Starting Asakusa Hadoop:"
-echo " JRID: $_OPT_JRID"
-echo " App Library: $_JQ_APP_LIB"
-echo " Batch ID: $_OPT_BATCH_ID"
-echo " Flow ID: $_OPT_FLOW_ID"
-echo " Execution ID: $_OPT_EXECUTION_ID"
-echo " Class: $_OPT_CLASS_NAME"
+echo " Hadoop Command: $HADOOP_CMD"
+echo " JRID: $_OPT_JRID"
+echo " App Library: $_JQ_APP_LIB"
+echo " Batch ID: $_OPT_BATCH_ID"
+echo " Flow ID: $_OPT_FLOW_ID"
+echo " Execution ID: $_OPT_EXECUTION_ID"
+echo " Class: $_OPT_CLASS_NAME"
echo " Hadoop Properties:"
echo " $_OPT_HADOOP_TMP_DIR"
echo " $_OPT_MAPRED_LOCAL_DIR"
echo " $_OPT_MAPRED_SYSTEM_DIR"
echo " $_OPT_MAPREDUCE_JOBTRACKER_STAGING_ROOT_DIR"
echo " $_OPT_MAPRED_TEMP_DIR"
-"$HADOOP_HOME/bin/hadoop" jar \
+"$HADOOP_CMD" jar \
"$_JQ_RUNTIME_LIB" \
"$_JQ_TOOL_LAUNCHER" \
"$_OPT_CLASS_NAME" \
-conf "$_JQ_PLUGIN_CONF" \
- -libjars "$_JQ_LIBJAR" \
+ -libjars "$_JQ_LIBJARS" \
-D "com.asakusafw.user=$USER" \
-D "com.asakusafw.executionId=$_OPT_EXECUTION_ID" \
-D "com.asakusafw.batchArgs=$_OPT_BATCH_ARGUMENTS" \
@@ -160,7 +148,7 @@ else
echo " Launcher: $_JQ_TOOL_LAUNCHER" 1>&2
echo " Stage Class: $_OPT_CLASS_NAME" 1>&2
echo "Configuration: -conf $_JQ_PLUGIN_CONF" 1>&2
- echo " Libraries: -libjars $_JQ_LIBJAR" 1>&2
+ echo " Libraries: -libjars $_JQ_LIBJARS" 1>&2
echo "Built-in Prop: -D com.asakusafw.user=$USER" 1>&2
echo "Built-in Prop: -D com.asakusafw.executionId=$_OPT_EXECUTION_ID" 1>&2
echo "Built-in Prop: -D com.asakusafw.batchArgs=$_OPT_BATCH_ARGUMENTS" 1>&2
View
21 ...ice-project/asakusa-jobqueue-server/src/main/dist/jobqueue-hadoop/libexec/validate-env.sh
@@ -0,0 +1,21 @@
+#
+# Copyright 2011-2012 Asakusa Framework Team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ "$ASAKUSA_HOME" = "" ]
+then
+ echo '$ASAKUSA_HOME'" is not defined" 1>&2
+ exit 1
+fi

0 comments on commit 3b6041f

Please sign in to comment.
Something went wrong with that request. Please try again.