Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

implement backends in python and javascript.

  • Loading branch information...
commit 78850822eae27c86c2a60ce76d487a8ee1a66435 1 parent db97d53
@gdusbabek gdusbabek authored
View
25 README.textile
@@ -23,10 +23,33 @@ Flewton uses "log4j":http://logging.apache.org/log4j/1.2 for logging. By defaul
bc.
java -Dlog4j.configuration=file:///path/to/log4.props -jar flewton.jar
+h2. Implementing backends in other languages
+
+Backends can be implemented in several languages. We currently support:
+* Python
+* Javascript
+
+There are three ways you can tell Flewton about your external backends:
+1. Place them in @/etc/flewton@.
+2. Run flewton with @-Dflewton.backend_path@, placing your external backends in that directory.
+3. Include your backends in the classpath so they can be loaded as resources by the classloader.
+
+Identify your external backends in @flewton.cfg@ by using their entire names, e.g.: @js/my/AwesomeBackend.js@ or @py/my/CoolerBackend.py@. The default cfg that ships with flewton contains a few commented out examples, the source for which can also be found in the Flewton source code.
+
+h3. Python
+
+* Your class should extend @AbstractBackend@.
+* Your class should have an @__init__@ method that accepts an instance of @HierarchicalConfiguration@.
+* Module names must begin with '@py/@' to be properly identified as python. E.g.: @py/my/module/path/Foo.py@
+
+h3. Javascript
+
+* Your javascript should export two methods: @init(config)@ and @write(record)@. The class types for @config@ and @record@ are the same as they would be in Java.
+
h2. Known Issues
* Currently only Netflow v5 is supported, but additional Netflow formats are possible by implementing decoder classes as @com.rackspace.flewton.RecordvN@ (where @N@ is the version). See @com.rackspace.flewton.Recordv5@ for an example. Patches welcome.
h2. About
-Flewton was developed by "Gary Dusbabek":mailto:gary.dusbabek@rackspace.com and "Eric Evans":mailto:eevans@rackspace.com and open-source by "Rackspace":http://www.rackspace.com.
+Flewton was developed by "Gary Dusbabek":mailto:gary.dusbabek@rackspace.com and "Eric Evans":mailto:eevans@rackspace.com and made open-source by "Rackspace":http://www.rackspace.com.
View
2  TODO
@@ -1,3 +1,3 @@
-* Eliminate dependency on com.eaio.uuid.UUIDGen (c.r.f.u.UUIDGen?).
* Include a (tested )v9 protocol decoder.
+* Unit tests for ExtBackendFactory that exercise loading from external directories.
View
14 flewton.cfg
@@ -5,6 +5,8 @@ listenPort = 9996
backendClass = com/rackspace/flewton/backend/NullBackend
; backendClass = com/rackspace/flewton/backend/cassandra/UsageBackend
; backendClass = com/rackspace/flewton/backend/TopTalkersBackend
+; backendClass = py/PyLoggingBackend.py
+; backendClass = js/JsLoggingBackend.js
; TopTalkersBackend ------------------------------------------------------
;
@@ -57,3 +59,15 @@ backendClass = com/rackspace/flewton/backend/NullBackend
;network = 11.11.11.0/20
;network = 22.22.22.0/24
;network = 33.33.33.0/19
+
+; PyLoggingBackend -------------------------------------------------------
+; There is no configuration, as this is a python example
+; ------------------------------------------------------------------------
+;[py/PyLoggingBackend.py]
+;foo = bar
+
+; JsLoggingBackend -------------------------------------------------------
+; There is no configuration, as this is a javascript example
+; ------------------------------------------------------------------------
+;[js/JsLoggingBackend.js]
+;foo = bar
View
BIN  lib/junit-4.6.jar
Binary file not shown
View
BIN  lib/jython.jar
Binary file not shown
View
8 src/com/rackspace/flewton/CollectorHandler.java
@@ -33,6 +33,7 @@
import java.util.ArrayList;
import java.util.List;
+import com.rackspace.flewton.backend.IBackend;
import org.apache.commons.configuration.HierarchicalConfiguration;
import org.jboss.netty.buffer.ChannelBuffer;
import org.jboss.netty.channel.ChannelHandlerContext;
@@ -42,13 +43,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.rackspace.flewton.backend.AbstractBackend;
import com.rackspace.flewton.backend.NullBackend;
public class CollectorHandler extends SimpleChannelHandler {
private static final Logger logger = LoggerFactory.getLogger(CollectorHandler.class);
private static final boolean logUnhandledVersions = Boolean.parseBoolean(System.getProperty("flewton.log_unhandled_versions", "false"));
- private static List<AbstractBackend> backEnds = new ArrayList<AbstractBackend>();
+ private static List<IBackend> backEnds = new ArrayList<IBackend>();
private static final int HEX_LENGTH = 16;
static {
@@ -84,7 +84,7 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
// Send record to backends
if (record != null)
- for (AbstractBackend backend : backEnds)
+ for (IBackend backend : backEnds)
backend.write(record);
}
@@ -132,7 +132,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
//e.getChannel().close();
}
- public static void setBackends(List<AbstractBackend> backends) {
+ public static void setBackends(List<IBackend> backends) {
backEnds = backends;
}
}
View
17 src/com/rackspace/flewton/CollectorServer.java
@@ -35,6 +35,8 @@
import java.util.List;
import java.util.concurrent.Executors;
+import com.rackspace.flewton.backend.ExtBackendFactory;
+import com.rackspace.flewton.backend.IBackend;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.HierarchicalConfiguration;
import org.apache.commons.configuration.HierarchicalINIConfiguration;
@@ -65,15 +67,22 @@
};
}
- private static List<AbstractBackend> createBackends(String[] backendNames, HierarchicalINIConfiguration config) throws ConfigError {
- List<AbstractBackend> backends = new ArrayList<AbstractBackend>();
+ private static List<IBackend> createBackends(String[] backendNames, HierarchicalINIConfiguration config) throws ConfigError {
+ List<IBackend> backends = new ArrayList<IBackend>();
for (String name : backendNames) {
try {
logger.info("Adding backend: {}", name);
- Class<?> backendClass = Class.forName(name.replace('/', '.'));
+ IBackend backend = null;
SubnodeConfiguration subConfig = config.getSection(name.replace('.', '/'));
- AbstractBackend backend = (AbstractBackend)backendClass.getConstructor(HierarchicalConfiguration.class).newInstance(subConfig);
+ if (name.startsWith("py/"))
+ backend = ExtBackendFactory.createPythonBackend(name, config);
+ else if (name.startsWith("js/"))
+ backend = ExtBackendFactory.createJavascriptBackend(name, config);
+ else {
+ Class<?> backendClass = Class.forName(name.replace('/', '.'));
+ backend = (AbstractBackend)backendClass.getConstructor(HierarchicalConfiguration.class).newInstance(subConfig);
+ }
backends.add(backend);
} catch (ClassNotFoundException e) {
logger.error("Backend not found: {} (not in classpath?)", name);
View
4 src/com/rackspace/flewton/ConfigError.java
@@ -41,4 +41,8 @@
public ConfigError(String msg) {
super(msg);
}
+
+ public ConfigError(String msg, Throwable cause) {
+ super(msg, cause);
+ }
}
View
2  src/com/rackspace/flewton/backend/AbstractBackend.java
@@ -33,7 +33,7 @@
import com.rackspace.flewton.AbstractRecord;
-public abstract class AbstractBackend {
+public abstract class AbstractBackend implements IBackend {
public AbstractBackend(HierarchicalConfiguration config) {
}
View
117 src/com/rackspace/flewton/backend/ExtBackendFactory.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2010 Rackspace
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ * may be used to endorse or promote products derived from this software
+ * without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+package com.rackspace.flewton.backend;
+
+import com.rackspace.flewton.ConfigError;
+import org.apache.commons.configuration.HierarchicalConfiguration;
+import org.python.util.PythonInterpreter;
+
+import javax.script.Invocable;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+
+public class ExtBackendFactory {
+ private static PythonInterpreter pyInterpreter = null;
+
+ private static String[] backendSearchPaths = new ArrayList<String>() {{
+ if (System.getProperty("flewton.backend_path") != null)
+ add(System.getProperty("flewton.backend_path"));
+ add("/etc/flewton/backends");
+ }}.toArray(new String[0]);
+
+ /**
+ * creates a backend that is implemented in Python.
+ * @param modulePath location of the python module. Starts with "py/"
+ * @param config configuration.
+ */
+ public static IBackend createPythonBackend(String modulePath, HierarchicalConfiguration config) {
+ if (pyInterpreter == null)
+ pyInterpreter = new PythonInterpreter();
+ pyInterpreter.execfile(getExternalResource(modulePath));
+ String pyClassName = modulePath.substring(modulePath.lastIndexOf("/") + 1);
+ pyClassName = pyClassName.substring(0, pyClassName.indexOf("."));
+ String instanceName = "instance_of_" + pyClassName;
+ String configName = instanceName + "_config";
+ pyInterpreter.set(configName, config);
+ String def = instanceName + " = " + pyClassName + "(" + configName + ")";
+ pyInterpreter.exec(def);
+
+ Object impl = pyInterpreter.get(instanceName).__tojava__(AbstractBackend.class);
+ return (AbstractBackend)impl;
+ }
+
+ /**
+ * creates a backend implemented in Javascript.
+ * @param jsPath location of javascript file. Starts with "js/"
+ * @param config configuration
+ * @throws ConfigError if there are any problems with the javascript.
+ */
+ public static IBackend createJavascriptBackend(String jsPath, HierarchicalConfiguration config) throws ConfigError
+ {
+ ScriptEngineManager manager = new ScriptEngineManager();
+ ScriptEngine engine = manager.getEngineByName("JavaScript");
+ try {
+ Object result = engine.eval(new InputStreamReader(getExternalResource(jsPath)));
+ Invocable inv = (Invocable)engine;
+ // init() js function mimics a java constructor.
+ inv.invokeFunction("init", config);
+ IBackend backend = inv.getInterface(IBackend.class);
+ return backend;
+ } catch (ScriptException ex) {
+ throw new ConfigError(ex.getMessage(), ex);
+ } catch (NoSuchMethodException ex) {
+ throw new ConfigError(ex.getMessage(), ex);
+ }
+ }
+
+ private static InputStream getExternalResource(String name) {
+ // check all the search paths first.
+ for (String path : backendSearchPaths) {
+ File resource = new File(path, name);
+ if (resource.exists()) {
+ try {
+ return new FileInputStream(resource);
+ } catch (FileNotFoundException suppress) {
+ // we already verified the file exists. so this would be bad.
+ throw new RuntimeException(suppress);
+ }
+ }
+ }
+ // now try loading from a few classloaders.
+ return ExtBackendFactory.class.getClassLoader().getResourceAsStream(name);
+ }
+}
View
9 src/com/rackspace/flewton/backend/IBackend.java
@@ -0,0 +1,9 @@
+package com.rackspace.flewton.backend;
+
+
+import com.rackspace.flewton.AbstractRecord;
+
+/** intended mainly for scripting languages that cannot easily inherit from a java class. */
+public interface IBackend {
+ public void write(AbstractRecord record);
+}
View
48 src/js/JsLoggingBackend.js
@@ -0,0 +1,48 @@
+/*
+ Copyright (c) 2010 Rackspace
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the University nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGE.
+ */
+
+var System = java.lang.System;
+
+init = function init(config) {
+ System.out.println("This is the config");
+}
+
+write = function write(record) {
+ // can't figure out how to iterate through record.flows using for..in.
+ for (i = 0; i < record.flows.size(); i++) {
+ flow = record.flows.get(i);
+ // todo: figure out how to make use of java.lang.String.format().
+ System.out.println(
+ "JS Proof! source=" + flow.sourceAddr.getCanonicalHostName() + ":" + flow.sourcePort +
+ ", dest=" + flow.destAddr.getCanonicalHostName() + ":" + flow.destPort +
+ ", bytes=" + flow.numOctets
+ );
+ }
+
+}
View
45 src/py/PyLoggingBackend.py
@@ -0,0 +1,45 @@
+"""
+ Copyright (c) 2010 Rackspace
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ 1. Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ 3. Neither the name of the University nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ SUCH DAMAGE.
+"""
+
+#from org.apache.commons.configuration import HierarchicalConfiguration
+#from com.rackspace.flewton import AbstractRecord
+from com.rackspace.flewton.backend import AbstractBackend
+
+class PyLoggingBackend(AbstractBackend):
+ """Example of how to create a simple backend in python."""
+ def __init__(self, config):
+ AbstractBackend.__init__(self, config)
+
+ def write(self, record):
+ for flow in record.flows:
+ print "Python Proof! source=%s:%d, dest=%s:%d, bytes=%d" % (flow.sourceAddr.getCanonicalHostName(),
+ flow.sourcePort,
+ flow.destAddr.getCanonicalHostName(),
+ flow.destPort,
+ flow.numOctets)
Please sign in to comment.
Something went wrong with that request. Please try again.