Permalink
Browse files

initial commit

  • Loading branch information...
eevans authored and flewton committed Sep 9, 2010
0 parents commit b5d19c1daf0855259fa04f61d5d76c170eca1e46
Showing with 2,171 additions and 0 deletions.
  1. +5 −0 .gitignore
  2. +26 −0 COPYING
  3. +32 −0 README.textile
  4. +3 −0 TODO
  5. +105 −0 build.xml
  6. +59 −0 flewton.cfg
  7. BIN lib/cassandra-thrift-0.7.0.jar
  8. BIN lib/commons-collections-3.2.1.jar
  9. BIN lib/commons-configuration-1.6.jar
  10. BIN lib/commons-lang-2.4.jar
  11. BIN lib/commons-logging-1.1.1.jar
  12. BIN lib/commons-pool-1.5.4.jar
  13. BIN lib/libthrift-0.5.jar
  14. BIN lib/log4j-1.2.16.jar
  15. BIN lib/netty-3.2.2.Final.jar
  16. BIN lib/slf4j-api-1.6.1.jar
  17. BIN lib/slf4j-log4j12-1.6.1.jar
  18. BIN lib/sources/commons-collections-3.2.1-sources.jar
  19. BIN lib/sources/commons-configuration-1.6-sources.jar
  20. BIN lib/sources/commons-lang-2.4-sources.jar
  21. BIN lib/sources/commons-logging-1.1.1-sources.jar
  22. BIN lib/sources/commons-pool-1.5.4-sources.jar
  23. BIN lib/sources/log4j-1.2.16-sources.jar
  24. BIN lib/sources/netty-3.2.2.Final-sources.jar
  25. BIN lib/sources/slf4j-api-1.6.1-sources.jar
  26. BIN lib/sources/slf4j-log4j12-1.6.1-sources.jar
  27. BIN lib/sources/uuid-3.2-sources.jar
  28. BIN lib/uuid-3.2.jar
  29. +43 −0 src/com/rackspace/flewton/AbstractRecord.java
  30. +136 −0 src/com/rackspace/flewton/CollectorHandler.java
  31. +168 −0 src/com/rackspace/flewton/CollectorServer.java
  32. +44 −0 src/com/rackspace/flewton/ConfigError.java
  33. +48 −0 src/com/rackspace/flewton/CorruptDatagram.java
  34. +52 −0 src/com/rackspace/flewton/Flow.java
  35. +124 −0 src/com/rackspace/flewton/Recordv5.java
  36. +42 −0 src/com/rackspace/flewton/backend/AbstractBackend.java
  37. +58 −0 src/com/rackspace/flewton/backend/LoggingBackend.java
  38. +49 −0 src/com/rackspace/flewton/backend/NullBackend.java
  39. +135 −0 src/com/rackspace/flewton/backend/TopTalkersBackend.java
  40. +211 −0 src/com/rackspace/flewton/backend/cassandra/CumulativeUsageBackend.java
  41. +151 −0 src/com/rackspace/flewton/backend/cassandra/ThriftClientPool.java
  42. +34 −0 src/com/rackspace/flewton/backend/cassandra/ThriftClientPoolMBean.java
  43. +265 −0 src/com/rackspace/flewton/backend/cassandra/UsageBackend.java
  44. +118 −0 src/com/rackspace/flewton/util/HostResolver.java
  45. +130 −0 src/com/rackspace/flewton/util/UUIDGen.java
  46. +6 −0 src/log4j.properties
  47. +127 −0 test/com/rackspace/flewton/test/SendFauxFlowsv5.java
@@ -0,0 +1,5 @@
+.classpath
+.project
+build/
+.settings
+dist/
26 COPYING
@@ -0,0 +1,26 @@
+Copyright (c) 2010 Rackspace
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+3. Neither the name of the University nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
@@ -0,0 +1,32 @@
+h1. Flewton
+
+Flewton is an extensible Netflow collector.
+
+h2. Configuration
+
+Refer to the comments in the sample configuration shipped with Flewton.
+
+h2. Running
+
+In a nutshell:
+
+bc.
+java [options] -jar flewton.jar
+
+By default, Flewton will search for its configuration file as @/etc/flewton/flewton.cfg@, @/etc/flewton.cfg@, and @./flewton.cfg@, using the first one found. You can specify a path to your configuration using the @flewton.config@ system property, for example:
+
+bc.
+java -Dflewton.config=/usr/local/etc/flewton.cfg -jar flewton.jar
+
+Flewton uses "log4j":http://logging.apache.org/log4j/1.2 for logging. By default, the log level is @INFO@ and output is sent to stdout. To customize logging, create your own log4j.properties and set the @log4j.configuration@ system property:
+
+bc.
+java -Dlog4j.configuration=file:///path/to/log4.props -jar flewton.jar
+
+h2. Known Issues
+
+* Currently only Netflow v5 is supported, but additional Netflow formats are possible by implementing decoder classes as @com.rackspace.flewton.RecordvN@ (where @N@ is the version). See @com.rackspace.flewton.Recordv5@ for an example. Patches welcome.
+
+h2. About
+
+Flewton was developed by "Gary Dusbabek":mailto:gary.dusbabek@rackspace.com and "Eric Evans":mailto:eevans@rackspace.com and open-source by "Rackspace":http://www.rackspace.com.
3 TODO
@@ -0,0 +1,3 @@
+
+* Eliminate dependency on com.eaio.uuid.UUIDGen (c.r.f.u.UUIDGen?).
+* Include a (tested )v9 protocol decoder.
105 build.xml
@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<project basedir="." default="build" name="flewton">
+ <property environment="env" />
+ <property name="basedir" value="." />
+ <property name="name" value="flewton" />
+ <property name="version" value="1.0" />
+ <property file="build.properties" />
+ <property name="build.dir" value="${basedir}/build" />
+ <property name="build.dir.classes" value="${basedir}/build/classes" />
+ <property name="src.dir" value="${basedir}/src" />
+ <property name="lib.dir" value="${basedir}/lib" />
+ <property name="debuglevel" value="source,lines,vars" />
+ <property name="flewton.config" value="flewton.cfg" />
+ <property name="dist.dir" value="${basedir}/dist" />
+
+ <!-- Duplicates the search order in CollectorServer.configSearchPaths -->
+ <condition property="flewton.config" value="/etc/flewton.cfg">
+ <available file="/etc/flewton.cfg" />
+ </condition>
+ <condition property="flewton.config" value="/etc/flewton/flewton.cfg">
+ <available file="/etc/flewton/flewton.cfg" />
+ </condition>
+
+ <path id="java.classpath">
+ <fileset dir="${lib.dir}">
+ <include name="*.jar" />
+ </fileset>
+ </path>
+
+ <pathconvert property="manifest.classpath" pathsep=" ">
+ <path refid="java.classpath" />
+ <mapper>
+ <chainedmapper>
+ <flattenmapper />
+ <globmapper from="*.jar" to="lib/*.jar" />
+ </chainedmapper>
+ </mapper>
+ </pathconvert>
+
+ <target name="setup">
+ <mkdir dir="${build.dir.classes}" />
+ </target>
+
+ <target name="build" depends="setup">
+ <javac destdir="${build.dir.classes}"
+ includeantruntime="false"
+ debug="true"
+ debuglevel="${debuglevel}">
+ <src path="${src.dir}" />
+ <classpath refid="java.classpath" />
+ </javac>
+ <copy file="${src.dir}/log4j.properties"
+ tofile="${build.dir.classes}/log4j.properties" />
+ </target>
+
+ <target name="jar" depends="build">
+ <mkdir dir="${dist.dir}"/>
+ <jar jarfile="${dist.dir}/${name}-${version}.jar"
+ basedir="${build.dir.classes}">
+ <manifest>
+ <attribute name="Implementation-Title" value="Flewton" />
+ <attribute name="Main-Class"
+ value="com.rackspace.flewton.CollectorServer" />
+ <attribute name="Class-Path" value="${manifest.classpath}" />
+ </manifest>
+ </jar>
+ </target>
+
+ <target name="dist" depends="jar">
+ <copy todir="${dist.dir}" file="README.textile" />
+ <copy todir="${dist.dir}" file="flewton.cfg" />
+ <copy todir="${dist.dir}/lib" flatten="true">
+ <fileset dir="${basedir}/lib/" excludes="sources/" />
+ </copy>
+ </target>
+
+ <target name="release" depends="dist">
+ <tar compression="gzip" longfile="gnu"
+ destfile="${dist.dir}/${name}-${version}.tar.gz">
+ <tarfileset dir="${dist.dir}" prefix="${name}-${version}">
+ <include name="**" />
+ <exclude name="*.tar.gz" />
+ </tarfileset>
+ </tar>
+ </target>
+
+ <target name="run" depends="dist">
+ <java jar="${dist.dir}/${name}-${version}.jar" fork="true">
+ <classpath>
+ <path refid="java.classpath" />
+ <pathelement location="${build.dir.classes}" />
+ </classpath>
+ <sysproperty key="flewton.config" value="${flewton.config}" />
+ </java>
+ </target>
+
+ <target name="clean">
+ <delete dir="${build.dir}" />
+ <delete dir="${dist.dir}" />
+ </target>
+</project>
+
+<!--
+vi: sw=4 ts=4 tw=0 et
+-->
@@ -0,0 +1,59 @@
+; UDP port to bind to for Netflow
+listenPort = 9996
+
+; Backends
+backendClass = com/rackspace/flewton/backend/NullBackend
+; backendClass = com/rackspace/flewton/backend/cassandra/UsageBackend
+; backendClass = com/rackspace/flewton/backend/TopTalkersBackend
+
+; TopTalkersBackend ------------------------------------------------------
+;
+; Periodically log a list of "Top Talkers".
+;
+; Config Directives
+; - maxEntries: the maximum number of "talkers" to track.
+; - intervalSecs: the log/reset period in seconds.
+; - network: list of (CIDR) networks found on our network
+; ------------------------------------------------------------------------
+;[com/rackspace/flewton/backend/TopTalkersBackend]
+;maxEntries = 1000
+;intervalSecs = 3600
+;network = 11.11.11.0/20
+;network = 22.22.22.0/24
+;network = 33.33.33.0/19
+
+; UsageBackend -----------------------------------------------------------
+;
+; Write per-host byte counts to a Cassandra cluster.
+;
+; This backend uses a configured list of networks to create an Us/Them
+; distinction in order to determine traffic direction. For example, when
+; the source is not in our network list (Them), and the destination is
+; (Us), the flow is deemed to be incoming and written to the column family
+; specified by wanInCf.
+;
+; Entries are written to Cassandra using a key composed of the address
+; in bytes, with one column for each flow recorded. Column names are
+; a type-1 (time-based) UUID and column values are the number of bytes as
+; an integer.
+;
+; Config Directives
+; - storageNode: list of Cassandra nodes to write to
+; - keypspace: Cassandra keyspace
+; - wanInCf: name of the column family to write incoming bytes to
+; - wanOutCf: name of the column family to write outgoing bytes to
+; - lanCf: name of the column family to write local bytes to
+; - columnTTLSecs: number of seconds to retain columns for
+; - network: list of (CIDR) networks found on our network
+; ------------------------------------------------------------------------
+;[com/rackspace/flewton/backend/cassandra/UsageBackend]
+;storageNode = n1.example.com:9160
+;storageNode = n2.example.com:9160
+;keyspace = Flewton
+;wanInCf = WANIn
+;wanOutCf = WANOut
+;lanCf = LAN
+;columnTTLSecs = 604800
+;network = 11.11.11.0/20
+;network = 22.22.22.0/24
+;network = 33.33.33.0/19
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2010 Rackspace
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package com.rackspace.flewton;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+
+public abstract class AbstractRecord {
+ public List<Flow> flows = new ArrayList<Flow>();
+
+ public AbstractRecord(ChannelBuffer buffer) {
+
+ }
+}
Oops, something went wrong.

0 comments on commit b5d19c1

Please sign in to comment.