Skip to content
Browse files

Few small bug fixes and consistency fixes for pipeline code.

git-svn-id: https://fourtwosix.jira.com/svn/TXPETE/trunk@13219 e6edf6fb-f266-4316-afb4-e53d95876a76
  • Loading branch information...
1 parent c932722 commit a730033fb0a601502dbe29e66669e5ecb918b311 ihelmke committed Jun 29, 2011
Showing with 5 additions and 2 deletions.
  1. +5 −2 pipeline/src/main/java/org/sleuthkit/hadoop/pipeline/Pipeline.java
View
7 pipeline/src/main/java/org/sleuthkit/hadoop/pipeline/Pipeline.java
@@ -29,6 +29,9 @@
import com.lightboxtechnologies.spectrum.HBaseTables;
import com.lightboxtechnologies.spectrum.HDFSArchiver;
+/** Runs all of the post-ingest tasks on the hadoop cloud, one after another.
+ * TODO: we could parallelize much of the things that happen here, as although
+ * they run in sequence, many of them are not dependent on each other to run. */
public class Pipeline {
// A file containing lines of text, each of which represents a regex.
public static final String GREP_KEYWORDS = "/texaspete/regexes";
@@ -55,12 +58,12 @@ public static void main(String[] argv) throws Exception {
boolean filesToSequence = (SequenceFsEntryText.runPipeline(seqDumpDirectory, imageID, friendlyName));
if (filesToSequence) {
TokenizeAndVectorizeDocuments.runPipeline(seqDumpDirectory, tokenDumpDirectory, vectorDumpDirectory);
- ClusterDocumentsJob.runPipeline(vectorDumpDirectory + "/tfidf-vectors/", clusterDumpDirectory, dictionaryDumpDirectory, .65, .65, imageID, friendlyName);
+ ClusterDocumentsJob.runPipeline(vectorDumpDirectory + "/tfidf-vectors/", clusterDumpDirectory, dictionaryDumpDirectory, .65, .65, imageID, friendlyName, prefix);
}
GrepReportGenerator.runPipeline(GREP_KEYWORDS, imageID, friendlyName, prefix);
- CrossImageScorerJob.runPipeline(prefix, imageID);
+ CrossImageScorerJob.runPipeline(prefix, imageID, friendlyName);
HDFSArchiver.runPipeline(prefix + "/reports", prefix + "/reports.zip");
}

0 comments on commit a730033

Please sign in to comment.
Something went wrong with that request. Please try again.