Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
Checking mergeability… Don't worry, you can still create the pull request.
  • 3 commits
  • 4 files changed
  • 0 commit comments
  • 1 contributor
View
2  Makefile
@@ -5,7 +5,7 @@ PIG_JAR=lib/packetpig.jar
all: $(PIG_JAR) spam_deletion
$(PIG_JAR): $(SRCS)
- cd lib/packetpig && mvn compile package && cp target/packetpig-*.jar ../../$(PIG_JAR)
+ cd lib/packetpig && mvn compile package && cp target/packetpig-*-with-dependencies.jar ../../$(PIG_JAR)
spam_deletion:
@rm -f pig_*.log
View
31 lib/packetpig/pom.xml
@@ -42,6 +42,11 @@
<version>1.4.0</version>
</dependency>
<dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <version>1.4.0</version>
+ </dependency>
+ <dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
@@ -67,4 +72,30 @@
<version>2.0.6</version>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ <archive>
+ <manifest>
+ <mainClass/>
+ </manifest>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
View
15 lib/packetpig/src/main/java/com/packetloop/packetpig/loaders/pcap/PcapStreamWriter.java
@@ -1,20 +1,19 @@
package com.packetloop.packetpig.loaders.pcap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.io.IOUtils;
-
import java.io.IOException;
import java.io.InputStream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+
public class PcapStreamWriter implements Runnable {
private Process process;
- private FSDataInputStream fsdis;
+ private InputStream is;
private Configuration config;
- public PcapStreamWriter(Configuration config, Process process, FSDataInputStream fsdis) {
+ public PcapStreamWriter(Configuration config, Process process, InputStream is) {
this.process = process;
- this.fsdis = fsdis;
+ this.is = is;
this.config = config;
}
@@ -23,7 +22,7 @@ public void run() {
System.err.println("copying fsdis to process...");
try {
- IOUtils.copyBytes(fsdis, process.getOutputStream(), config);
+ IOUtils.copyBytes(is, process.getOutputStream(), config);
} catch (IOException ignored) {
try {
InputStream stderr = process.getErrorStream();
View
39 lib/packetpig/src/main/java/com/packetloop/packetpig/loaders/pcap/StreamingPcapRecordReader.java
@@ -1,21 +1,23 @@
package com.packetloop.packetpig.loaders.pcap;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.zip.GZIPInputStream;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.bzip2.CBZip2InputStream;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import java.io.*;
-
public abstract class StreamingPcapRecordReader extends PcapRecordReader {
private TaskAttemptContext context;
protected Thread thread;
protected Process process;
- private long pos;
- private long len;
- private FSDataInputStream fsdis;
+ private InputStream is;
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
super.initialize(split, context);
@@ -30,9 +32,16 @@ protected BufferedReader streamingProcess(String cmd, String path) throws IOExce
FileSystem fs = FileSystem.get(config);
Path dfsPath = new Path(path);
- fsdis = fs.open(dfsPath);
- len = fs.getLength(dfsPath);
- pos = 0;
+ is = fs.open(dfsPath);
+ String name = dfsPath.getName().toLowerCase();
+ if(name.endsWith(".gz"))
+ {
+ is = new GZIPInputStream(is);
+ }
+ else if(name.endsWith(".bz") || name.endsWith(".bz2"))
+ {
+ is = new CBZip2InputStream(is);
+ }
/*
File out = File.createTempFile("packetpig", "stream");
@@ -51,7 +60,7 @@ protected BufferedReader streamingProcess(String cmd, String path) throws IOExce
ignore(process.getErrorStream());
// pipe from pcap stream into snort
- PcapStreamWriter writer = new PcapStreamWriter(config, process, fsdis);
+ PcapStreamWriter writer = new PcapStreamWriter(config, process, is);
thread = new Thread(writer);
thread.start();
@@ -65,15 +74,7 @@ protected void ignore(InputStream errorStream) {
@Override
public float getProgress() {
- try {
- pos = fsdis.getPos();
- } catch (IOException ignored) {
- pos = 0;
- }
-
- float progress = (float)pos / (float)len;
- System.err.println(progress);
- return progress;
+ return 0.0f;
}
}

No commit comments for this range

Something went wrong with that request. Please try again.