[INLONG-635]Add inlong-dataproxy subdirectory

inlong-dataproxy/bin/flume-ng

@@ -0,0 +1,379 @@
+#!/bin/bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+################################
+# constants
+################################
+
+FLUME_AGENT_CLASS="org.apache.flume.node.Application"
+FLUME_AVRO_CLIENT_CLASS="org.apache.flume.client.avro.AvroCLIClient"
+FLUME_VERSION_CLASS="org.apache.flume.tools.VersionInfo"
+
+CLEAN_FLAG=1
+################################
+# functions
+################################
+
+info() {
+  if [ ${CLEAN_FLAG} -ne 0 ]; then
+    local msg=$1
+    echo "Info: $msg" >&2
+  fi
+}
+
+warn() {
+  if [ ${CLEAN_FLAG} -ne 0 ]; then
+    local msg=$1
+    echo "Warning: $msg" >&2
+  fi
+}
+
+error() {
+  local msg=$1
+  local exit_code=$2
+
+  echo "Error: $msg" >&2
+
+  if [ -n "$exit_code" ] ; then
+    exit $exit_code
+  fi
+}
+
+# If avail, add Hadoop paths to the FLUME_CLASSPATH and to the
+# FLUME_JAVA_LIBRARY_PATH env vars.
+# Requires Flume jars to already be on FLUME_CLASSPATH.
+add_hadoop_paths() {
+  local HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" \
+      which hadoop 2>/dev/null)
+
+  if [ -f "${HADOOP_IN_PATH}" ]; then
+    info "Including Hadoop libraries found via ($HADOOP_IN_PATH) for HDFS access"
+
+    # determine hadoop java.library.path and use that for flume
+    local HADOOP_CLASSPATH=""
+    local HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$FLUME_CLASSPATH" \
+        ${HADOOP_IN_PATH} org.apache.flume.tools.GetJavaProperty \
+        java.library.path)
+
+    # look for the line that has the desired property value
+    # (considering extraneous output from some GC options that write to stdout)
+    # IFS = InternalFieldSeparator (set to recognize only newline char as delimiter)
+    IFS=$'\n'
+    for line in $HADOOP_JAVA_LIBRARY_PATH; do
+      if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
+        HADOOP_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
+        break
+      fi
+    done
+    unset IFS
+
+    if [ -n "${HADOOP_JAVA_LIBRARY_PATH}" ]; then
+      FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HADOOP_JAVA_LIBRARY_PATH"
+    fi
+
+    # determine hadoop classpath
+    HADOOP_CLASSPATH=$($HADOOP_IN_PATH classpath)
+
+    # hack up and filter hadoop classpath
+    local ELEMENTS=$(sed -e 's/:/ /g' <<<${HADOOP_CLASSPATH})
+    local ELEMENT
+    for ELEMENT in $ELEMENTS; do
+      local PIECE
+      for PIECE in $(echo $ELEMENT); do
+        if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
+          info "Excluding $PIECE from classpath"
+          continue
+        else
+          FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
+        fi
+      done
+    done
+
+  fi
+}
+add_HBASE_paths() {
+  local HBASE_IN_PATH=$(PATH="${HBASE_HOME}/bin:$PATH" \
+      which hbase 2>/dev/null)
+
+  if [ -f "${HBASE_IN_PATH}" ]; then
+    info "Including HBASE libraries found via ($HBASE_IN_PATH) for HBASE access"
+
+    # determine HBASE java.library.path and use that for flume
+    local HBASE_CLASSPATH=""
+    local HBASE_JAVA_LIBRARY_PATH=$(HBASE_CLASSPATH="$FLUME_CLASSPATH" \
+        ${HBASE_IN_PATH} org.apache.flume.tools.GetJavaProperty \
+        java.library.path)
+
+    # look for the line that has the desired property value
+    # (considering extraneous output from some GC options that write to stdout)
+    # IFS = InternalFieldSeparator (set to recognize only newline char as delimiter)
+    IFS=$'\n'
+    for line in $HBASE_JAVA_LIBRARY_PATH; do
+      if [[ $line =~ ^java\.library\.path=(.*)$ ]]; then
+        HBASE_JAVA_LIBRARY_PATH=${BASH_REMATCH[1]}
+        break
+      fi
+    done
+    unset IFS
+
+    if [ -n "${HBASE_JAVA_LIBRARY_PATH}" ]; then
+      FLUME_JAVA_LIBRARY_PATH="$FLUME_JAVA_LIBRARY_PATH:$HBASE_JAVA_LIBRARY_PATH"
+    fi
+
+    # determine HBASE classpath
+    HBASE_CLASSPATH=$($HBASE_IN_PATH classpath)
+
+    # hack up and filter HBASE classpath
+    local ELEMENTS=$(sed -e 's/:/ /g' <<<${HBASE_CLASSPATH})
+    local ELEMENT
+    for ELEMENT in $ELEMENTS; do
+      local PIECE
+      for PIECE in $(echo $ELEMENT); do
+        if [[ $PIECE =~ slf4j-(api|log4j12).*\.jar ]]; then
+          info "Excluding $PIECE from classpath"
+          continue
+        else
+          FLUME_CLASSPATH="$FLUME_CLASSPATH:$PIECE"
+        fi
+      done
+    done
+    FLUME_CLASSPATH="$FLUME_CLASSPATH:$HBASE_HOME/conf"
+
+  fi
+}
+
+set_LD_LIBRARY_PATH(){
+#Append the FLUME_JAVA_LIBRARY_PATH to whatever the user may have specified in
+#flume-env.sh
+  if [ -n "${FLUME_JAVA_LIBRARY_PATH}" ]; then
+    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${FLUME_JAVA_LIBRARY_PATH}"
+  fi
+}
+
+display_help() {
+  cat <<EOF
+Usage: $0 <command> [options]...
+
+commands:
+  help                  display this help text
+  agent                 run a Flume agent
+  avro-client           run an avro Flume client
+  version               show Flume version info
+
+global options:
+  --conf,-c <conf>      use configs in <conf> directory
+  --classpath,-C <cp>   append to the classpath
+  --dryrun,-d           do not actually start Flume, just print the command
+  -Dproperty=value      sets a JDK system property value
+
+agent options:
+  --conf-file,-f <file> specify a config file (required)
+  --name,-n <name>      the name of this agent (required)
+  --help,-h             display help text
+
+avro-client options:
+  --dirname <dir>       directory to stream to avro source
+  --host,-H <host>      hostname to which events will be sent (required)
+  --port,-p <port>      port of the avro source (required)
+  --filename,-F <file>  text file to stream to avro source [default: std input]
+  --headerFile,-R <file> headerFile containing headers as key/value pairs on each new line
+  --help,-h             display help text
+
+Note that if <conf> directory is specified, then it is always included first
+in the classpath.
+
+EOF
+}
+
+run_flume() {
+  local FLUME_APPLICATION_CLASS
+
+  if [ "$#" -gt 0 ]; then
+    FLUME_APPLICATION_CLASS=$1
+    shift
+  else
+    error "Must specify flume application class" 1
+  fi
+
+  if [ ${CLEAN_FLAG} -ne 0 ]; then
+    set -x
+  fi
+  $EXEC $JAVA_HOME/bin/java $JAVA_OPTS -cp "$FLUME_CLASSPATH" -Djava.util.logging.config.file="/data/work/osgoo/authcentertest/apache-flume-1.6.0-bin/conf/netty.properties" \
+      -Djava.library.path=$FLUME_JAVA_LIBRARY_PATH "$FLUME_APPLICATION_CLASS" $*
+}
+
+################################
+# main
+################################
+
+# set default params
+FLUME_CLASSPATH=""
+JAVA_OPTS="-Xmx10g -Xms10g -Xmn2g -XX:SurvivorRatio=6"
+#LD_LIBRARY_PATH=""
+
+opt_conf=""
+opt_classpath=""
+opt_dryrun=""
+
+mode=$1
+shift
+
+case "$mode" in
+  help)
+    display_help
+    exit 0
+    ;;
+  agent)
+    opt_agent=1
+    ;;
+  node)
+    opt_agent=1
+    warn "The \"node\" command is deprecated. Please use \"agent\" instead."
+    ;;
+  avro-client)
+    opt_avro_client=1
+    ;;
+  version)
+   opt_version=1
+   CLEAN_FLAG=0
+   ;;
+  *)
+    error "Unknown or unspecified command '$mode'"
+    echo
+    display_help
+    exit 1
+    ;;
+esac
+
+while [ -n "$*" ] ; do
+  arg=$1
+  shift
+
+  case "$arg" in
+    --conf|-c)
+      [ -n "$1" ] || error "Option --conf requires an argument" 1
+      opt_conf=$1
+      shift
+      ;;
+    --classpath|-C)
+      [ -n "$1" ] || error "Option --classpath requires an argument" 1
+      opt_classpath=$1
+      shift
+      ;;
+    --dryrun|-d)
+      opt_dryrun="1"
+      ;;
+    -D*)
+      opt_java_props="$opt_java_props $arg"
+      ;;
+    -X*)
+      opt_java_props="$opt_java_props $arg"
+      ;;
+    *)
+      args="$args $arg"
+      ;;
+  esac
+done
+
+# make opt_conf absolute
+if [[ -n "$opt_conf" && -d "$opt_conf" ]]; then
+  opt_conf=$(cd $opt_conf; pwd)
+fi
+
+# allow users to override the default env vars via conf/flume-env.sh
+if [ -z "$opt_conf" ]; then
+  warn "No configuration directory set! Use --conf <dir> to override."
+elif [ -f "$opt_conf/flume-env.sh" ]; then
+  info "Sourcing environment configuration script $opt_conf/flume-env.sh"
+  source "$opt_conf/flume-env.sh"
+fi
+
+# append command-line java options to stock or env script JAVA_OPTS
+if [ -n "${opt_java_props}" ]; then
+  JAVA_OPTS="${JAVA_OPTS} ${opt_java_props}"
+fi
+
+# prepend command-line classpath to env script classpath
+if [ -n "${opt_classpath}" ]; then
+  if [ -n "${FLUME_CLASSPATH}" ]; then
+    FLUME_CLASSPATH="${opt_classpath}:${FLUME_CLASSPATH}"
+  else
+    FLUME_CLASSPATH="${opt_classpath}"
+  fi
+fi
+
+if [ -z "${FLUME_HOME}" ]; then
+  FLUME_HOME=$(cd $(dirname $0)/..; pwd)
+fi
+
+# prepend $FLUME_HOME/lib jars to the specified classpath (if any)
+if [ -n "${FLUME_CLASSPATH}" ] ; then
+  FLUME_CLASSPATH="${FLUME_HOME}/lib/*:$FLUME_CLASSPATH"
+else
+  FLUME_CLASSPATH="${FLUME_HOME}/lib/*"
+fi
+
+# find java
+if [ -z "${JAVA_HOME}" ] ; then
+  warn "JAVA_HOME is not set!"
+  # Try to use Bigtop to autodetect JAVA_HOME if it's available
+  if [ -e /usr/libexec/bigtop-detect-javahome ] ; then
+    . /usr/libexec/bigtop-detect-javahome
+  elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ] ; then
+    . /usr/lib/bigtop-utils/bigtop-detect-javahome
+  fi
+
+  # Using java from path if bigtop is not installed or couldn't find it
+  if [ -z "${JAVA_HOME}" ] ; then
+    JAVA_DEFAULT=$(type -p java)
+    [ -n "$JAVA_DEFAULT" ] || error "Unable to find java executable. Is it in your PATH?" 1
+    JAVA_HOME=$(cd $(dirname $JAVA_DEFAULT)/..; pwd)
+  fi
+fi
+
+# look for hadoop libs
+#add_hadoop_paths
+#add_HBASE_paths
+
+# prepend conf dir to classpath
+if [ -n "$opt_conf" ]; then
+  FLUME_CLASSPATH="$opt_conf:$FLUME_CLASSPATH"
+fi
+
+set_LD_LIBRARY_PATH
+# allow dryrun
+EXEC="exec"
+if [ -n "${opt_dryrun}" ]; then
+  warn "Dryrun mode enabled (will not actually initiate startup)"
+  EXEC="echo"
+fi
+echo "LD_LIBRARY_PATH:$LD_LIBRARY_PATH"
+# finally, invoke the appropriate command
+if [ -n "$opt_agent" ] ; then
+  run_flume $FLUME_AGENT_CLASS $args
+elif [ -n "$opt_avro_client" ] ; then
+  run_flume $FLUME_AVRO_CLIENT_CLASS $args
+elif [ -n "${opt_version}" ] ; then
+  run_flume $FLUME_VERSION_CLASS $args
+else
+  error "This message should never appear" 1
+fi
+exit 0