Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
CHUKWA-605. Update directory structure to be aligned with Hadoop. (Er…
…ic Yang)

git-svn-id: https://svn.apache.org/repos/asf/incubator/chukwa/trunk@1206674 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
macroadster committed Nov 27, 2011
1 parent c36ce45 commit ba6e2a356d942fb78f894bf1d765a762f9ce55d2
Show file tree
Hide file tree
Showing 42 changed files with 293 additions and 314 deletions.
@@ -36,6 +36,8 @@ Trunk (unreleased changes)

IMPROVEMENTS

CHUKWA-605. Update directory structure to be aligned with Hadoop. (Eric Yang)

CHUKWA-601. Removed legacy mysql upgrade script. (Eric Yang)

CHUKWA-596. Updated schema retrieve REST API to use end time for scanning columns. (Eric Yang)
@@ -1,4 +1,4 @@
Chukwa 0.4 -- April 2010
Chukwa 0.5 -- April 2010

This is the second formal release of Chukwa, an Apache Hadoop subproject
dedicated to scalable log collection and processing. If you have large
@@ -10,7 +10,8 @@ The notes for this release are in docs/releasenotes.html
BUILDING CHUKWA

To build chukwa from source:
In the Chukwa root directory, say 'ant', and then 'cp build/*.jar build/*.war .'

mvn clean package

To check that things are ok, run 'ant test'. It should take roughly fifteen minutes.

@@ -27,7 +27,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# if no args specified, show usage
if [ $# = 0 ]; then
@@ -107,7 +107,7 @@ elif [ "$COMMAND" = "demux" ]; then
CLASS='org.apache.hadoop.chukwa.extraction.demux.DemuxManager'
PID='DemuxManager'
elif [ "$COMMAND" = "hicc" ]; then
WEB_SERVICE_COMMON=`ls ${CHUKWA_HOME}/webapps/hicc.war`
WEB_SERVICE_COMMON=`ls ${CHUKWA_HOME}/share/chukwa/webapps/hicc.war`
APP='hicc'
CLASS='org.apache.hadoop.chukwa.hicc.HiccWebServer'
CLASSPATH=${CLASSPATH}:${WEB_SERVICE_COMMON}:${HICC_JAR}
@@ -165,9 +165,9 @@ if [ "$1" = "stop" ]; then
else
# run command
if [ "$BACKGROUND" = "false" ]; then
${JAVA_HOME}/bin/java ${JAVA_OPT} -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_DATA_DIR=${CHUKWA_DATA_DIR} -DAPP=${APP} -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools} ${CLASS} $OPTS $@
${JAVA_HOME}/bin/java ${JAVA_OPT} -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_DATA_DIR=${CHUKWA_DATA_DIR} -DAPP=${APP} -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CLASSPATH}:${tools} ${CLASS} $OPTS $@
else
exec ${JAVA_HOME}/bin/java ${JAVA_OPT} -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_DATA_DIR=${CHUKWA_DATA_DIR} -DAPP=${APP} -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CORE}:${HADOOP_JAR}:${COMMON}:${tools} ${CLASS} $OPTS $@ &
exec ${JAVA_HOME}/bin/java ${JAVA_OPT} -Djava.library.path=${JAVA_LIBRARY_PATH} -DCHUKWA_HOME=${CHUKWA_HOME} -DCHUKWA_CONF_DIR=${CHUKWA_CONF_DIR} -DCHUKWA_LOG_DIR=${CHUKWA_LOG_DIR} -DCHUKWA_DATA_DIR=${CHUKWA_DATA_DIR} -DAPP=${APP} -Dlog4j.configuration=chukwa-log4j.properties -classpath ${CHUKWA_CONF_DIR}:${CLASSPATH}:${CHUKWA_CLASSPATH}:${tools} ${CLASS} $OPTS $@ &
fi
fi

@@ -20,26 +20,15 @@

# resolve links - $0 may be a softlink

this="$0"
while [ -h "$this" ]; do
ls=`ls -ld "$this"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '.*/.*' > /dev/null; then
this="$link"
else
this=`dirname "$this"`/"$link"
fi
done
this="${BASH_SOURCE-$0}"

# convert relative path to absolute path
bin=`dirname "$this"`
script=`basename "$this"`
bin=`cd "$bin"; pwd`
this="$bin/$script"

CHUKWA_LIBEXEC=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
script="$(basename -- "$this")"
this="$CHUKWA_PREFIX/$script"

# the root of the Chukwa installation
export CHUKWA_HOME=`dirname "$this"`/..
export CHUKWA_HOME=`pwd -P ${CHUKWA_LIBEXEC}/..`

#check to see if the conf dir is given as an optional argument
if [ $# -gt 1 ]
@@ -74,56 +63,22 @@ if [ -z ${CHUKWA_PID_DIR} ]; then
export CHUKWA_PID_DIR="${CHUKWA_HOME}/var/run"
fi

CHUKWA_VERSION=`cat ${CHUKWA_HOME}/bin/VERSION`
CHUKWA_VERSION=`cat ${CHUKWA_HOME}/share/chukwa/VERSION`

# Allow alternate conf dir location.
if [ -z "$CHUKWA_CONF_DIR" ]; then
CHUKWA_CONF_DIR="${CHUKWA_CONF_DIR:-$CHUKWA_HOME/conf}"
export CHUKWA_CONF_DIR=${CHUKWA_HOME}/conf
export CHUKWA_CONF_DIR="${CHUKWA_CONF_DIR:-$CHUKWA_HOME/etc/chukwa}"
fi

if [ -f "${CHUKWA_CONF_DIR}/chukwa-env.sh" ]; then
. "${CHUKWA_CONF_DIR}/chukwa-env.sh"
fi

if [ -d "${CHUKWA_HOME}/build/ivy/lib/chukwa/common" ]; then
COMMON=`ls ${CHUKWA_HOME}/lib/*.jar ${CHUKWA_HOME}/build/ivy/lib/chukwa/common/*.jar`
else
COMMON=`ls ${CHUKWA_HOME}/lib/*.jar`
fi
export COMMON=`echo ${COMMON} | sed 'y/ /:/'`
CHUKWA_CLASSPATH="${CHUKWA_HOME}/share/chukwa/*:${CHUKWA_HOME}/share/chukwa/lib/*"

export CHUKWA_CORE=${CHUKWA_HOME}/chukwa-core-${CHUKWA_VERSION}.jar
export CHUKWA_AGENT=${CHUKWA_HOME}/chukwa-agent-${CHUKWA_VERSION}.jar
export HICC_JAR=${CHUKWA_HOME}/hicc.war
export CURRENT_DATE=`date +%Y%m%d%H%M`

# Deprecated configuration for loading data to database.
export DATACONFIG=${CHUKWA_CONF_DIR}/mdl.xml

if [ -z ${HADOOP_JAR} ]; then
if [ -z ${HADOOP_HOME} ]; then
if [ -d ${CHUKWA_HOME}/hadoopjars ]; then
echo "WARNING: neither HADOOP_HOME nor HADOOP_JAR is set we we are reverting to defaults in $CHUKWA_HOME/hadoopjars dir"
export HADOOP_JAR=`ls ${CHUKWA_HOME}/hadoopjars/hadoop-*-core.jar`
else
echo "Please make sure hadoop-*-core.jar exists in ${CHUKWA_HOME}/hadoopjars"
exit -1
fi
else
if [ -d ${HADOOP_HOME} ]; then
export HADOOP_JAR=`ls ${HADOOP_HOME}/hadoop-*-core.jar`
if [ -z ${HADOOP_JAR} ]; then
echo "Please make sure hadoop-*-core.jar exists in ${HADOOP_HOME}"
exit -1
fi
fi
fi #end if no HADOOP_HOME
fi

if [ -z "$JAVA_HOME" ] ; then
echo ERROR! You forgot to set JAVA_HOME in conf/chukwa-env.sh
echo ERROR! You forgot to set JAVA_HOME in CHUKWA_CONF_DIR/chukwa-env.sh
fi

export JPS="ps ax"

@@ -39,7 +39,7 @@ fi
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# get arguments
startStop=$1
@@ -29,6 +29,6 @@ fi
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. $bin/chukwa-config.sh
. $bin/../libexec/chukwa-config.sh

exec "$bin/slaves.sh" --config $CHUKWA_CONF_DIR cd "$CHUKWA_HOME" \; "$bin/chukwa-daemon.sh" --config $CHUKWA_CONF_DIR "$@"
@@ -18,7 +18,7 @@ pid=$$

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh


pidFile=${CHUKWA_PID_DIR}/DailyChukwaRecordRolling.pid
@@ -38,7 +38,7 @@ pid=$$
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

while getopts ":c:d:m:n:t:" OPTION
do
@@ -18,7 +18,7 @@ pid=$$

bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

pidFile=${CHUKWA_PID_DIR}/HourlyChukwaRecordRolling.pid
if [ -f $pidFile ]; then
@@ -38,7 +38,7 @@ fi
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# If the slaves file is specified in the command line,
# then it takes precedence over the definition in
@@ -25,7 +25,7 @@ usage="Usage: start-agent.sh"
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# start chukwa agents
"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts agents start agent
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# start collectors
"$bin"/start-collectors.sh --config $CHUKWA_CONF_DIR
@@ -25,7 +25,7 @@ usage="Usage: start-collectors.sh"
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# start jetty collectors
"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts collectors start collector
@@ -19,7 +19,7 @@ bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
java=$JAVA_HOME/bin/java

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# start data processors
"$bin"/chukwa-daemon.sh --config $CHUKWA_CONF_DIR start archive
@@ -21,6 +21,6 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts agents stop agent
@@ -21,7 +21,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

"$bin"/stop-data-processors.sh --config $CHUKWA_CONF_DIR
"$bin"/stop-agents.sh --config $CHUKWA_CONF_DIR
@@ -21,6 +21,6 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

"$bin"/chukwa-daemons.sh --config $CHUKWA_CONF_DIR --hosts collectors stop collector
@@ -19,7 +19,7 @@ bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
java=$JAVA_HOME/bin/java

. "$bin"/chukwa-config.sh
. "$bin"/../libexec/chukwa-config.sh

# stop demux
pidFile=$CHUKWA_PID_DIR/DemuxManager.pid
File renamed without changes.
@@ -0,0 +1 @@
${TODO_ALERT_EMAIL}

This file was deleted.

@@ -24,7 +24,7 @@

<property>
<name>chukwaAgent.tags</name>
<value>cluster="@TODO-CLUSTER-NAME@"</value>
<value>cluster="${TODO_CLUSTER_NAME}"</value>
<description>The cluster's name for this agent</description>
</property>

@@ -34,7 +34,7 @@
<!-- LocalWriter parameters
<property>
<name>chukwaCollector.localOutputDir</name>
<value>@TODO-COLLECTORS-LOCAL-OUTPUT-DIR@</value>
<value>${TODO_COLLECTORS_LOCAL_OUTPUT_DIR}</value>
<description>Chukwa local data sink directory, see LocalWriter.java</description>
</property>
@@ -80,7 +80,7 @@ your hbase.zookeeper.quorum setting to the conf/ dir. -->

<property>
<name>writer.hdfs.filesystem</name>
<value>@TODO-COLLECTORS-NAMENODE@</value>
<value>${TODO_COLLECTORS_NAMENODE}</value>
<description>HDFS to dump to</description>
</property>

@@ -92,13 +92,13 @@ your hbase.zookeeper.quorum setting to the conf/ dir. -->

<property>
<name>chukwaCollector.rotateInterval</name>
<value>@TODO-COLLECTORS-ROTATEINTERVAL@</value>
<value>${TODO_COLLECTORS_ROTATEINTERVAL}</value>
<description>Chukwa rotate interval (ms)</description>
</property>

<property>
<name>chukwaCollector.isFixedTimeRotatorScheme</name>
<value>@TODO-COLLECTORS-ISFIXEDTIMEROTATORSCHEME</value>
<value>${TODO_COLLECTORS_ISFIXEDTIMEROTATORSCHEME}</value>
<description>A flag to indicate that the collector should close at a fixed
offset after every rotateInterval. The default value is false which uses
the default scheme where collectors close after regular rotateIntervals.
@@ -113,13 +113,13 @@ your hbase.zookeeper.quorum setting to the conf/ dir. -->

<property>
<name>chukwaCollector.fixedTimeIntervalOffset</name>
<value>@TODO-COLLECTORS-FIXEDTIMEINTERVALOFFSET@</value>
<value>${TODO_COLLECTORS_FIXEDTIMEINTERVALOFFSET}</value>
<description>Chukwa fixed time interval offset value (ms)</description>
</property>

<property>
<name>chukwaCollector.http.port</name>
<value>@TODO-COLLECTORS-PORT@</value>
<value>${TODO_COLLECTORS_PORT}</value>
<description>The HTTP port number the collector will listen on</description>
</property>

@@ -23,13 +23,13 @@

<property>
<name>chukwa.data.dir</name>
<value>@CHUKWA_DIR@</value>
<value>${CHUKWA_DIR}</value>
<description>Location of Chukwa data on HDFS</description>
</property>

<property>
<name>chukwa.tmp.data.dir</name>
<value>@CHUKWA_DIR@/temp</value>
<value>${CHUKWA_DIR}/temp</value>
<description>Location of Chukwa data on HDFS</description>
</property>

@@ -24,21 +24,21 @@
<!-- Chukwa Job parameters -->
<property>
<name>io.sort.mb</name>
<value>@TODO-DEMUX-IO-SORT-MB@</value>
<value>${TODO_DEMUX_IO_SORT_MB}</value>
<description>The total amount of buffer memory to use while sorting
files, in megabytes. By default, gives each merge stream 1MB, which
should minimize seeks.</description>
</property>

<property>
<name>fs.inmemory.size.mb</name>
<value>@TODO-DEMUX-FS-INMEMORY-SIZE_MB@</value>
<value>${TODO_DEMUX_FS_INMEMORY_SIZE_MB}</value>
<description>The size of the in-memory filsystem instance in MB</description>
</property>

<property>
<name>io.sort.factor</name>
<value>@TODO-DEMUX-IO-SORT-FACTOR@</value>
<value>${TODO_DEMUX_IO_SORT_FACTOR}</value>
<description>The number of streams to merge at once while sorting
files. This determines the number of open file handles.</description>
</property>

0 comments on commit ba6e2a3

Please sign in to comment.