Skip to content
Browse files

First stab at front end to Fisheye application that wraps RecordBreaker

  • Loading branch information...
1 parent b6e8e76 commit f63fc3b4906e661a6dc7e2ef309f154f3474bd24 @mikecafarella mikecafarella committed Feb 18, 2012
Showing with 8,419 additions and 134 deletions.
  1. +8 −2 bin/learnstructure
  2. +3 −0 ivy.xml
  3. +36 −0 src/java/com/cloudera/recordbreaker/analyzer/CrawlSummary.java
  4. +336 −129 src/java/com/cloudera/recordbreaker/analyzer/FSAnalyzer.java
  5. +84 −0 src/java/com/cloudera/recordbreaker/analyzer/FileSummary.java
  6. +39 −0 src/java/com/cloudera/recordbreaker/analyzer/FileSummaryData.java
  7. +60 −0 src/java/com/cloudera/recordbreaker/analyzer/SchemaSummary.java
  8. +30 −0 src/java/com/cloudera/recordbreaker/analyzer/SchemaSummaryData.java
  9. +50 −0 src/java/com/cloudera/recordbreaker/analyzer/TypeGuessSummary.java
  10. +56 −0 src/java/com/cloudera/recordbreaker/analyzer/TypeSummary.java
  11. +30 −0 src/java/com/cloudera/recordbreaker/analyzer/TypeSummaryData.java
  12. +0 −3 src/java/com/cloudera/recordbreaker/analyzer/UnknownTextDataDescriptor.java
  13. +9 −0 src/java/com/cloudera/recordbreaker/fisheye/AboutPage.java
  14. +55 −0 src/java/com/cloudera/recordbreaker/fisheye/FilePage.java
  15. +26 −0 src/java/com/cloudera/recordbreaker/fisheye/FilesPage.java
  16. +31 −0 src/java/com/cloudera/recordbreaker/fisheye/FiletypePage.java
  17. +27 −0 src/java/com/cloudera/recordbreaker/fisheye/FiletypesPage.java
  18. +107 −0 src/java/com/cloudera/recordbreaker/fisheye/FishEye.java
  19. +59 −0 src/java/com/cloudera/recordbreaker/fisheye/FishEyeWebApplication.java
  20. +10 −0 src/java/com/cloudera/recordbreaker/fisheye/Overview.java
  21. +34 −0 src/java/com/cloudera/recordbreaker/fisheye/SchemaPage.java
  22. +61 −0 src/java/com/cloudera/recordbreaker/fisheye/SchemasPage.java
  23. +31 −0 src/java/com/cloudera/recordbreaker/fisheye/SettingsPage.java
  24. +84 −0 src/web/AboutPage.html
  25. +122 −0 src/web/FilePage.html
  26. +103 −0 src/web/FilesPage.html
  27. +103 −0 src/web/FiletypesPage.html
  28. +103 −0 src/web/Overview.html
  29. +103 −0 src/web/SchemaPage.html
  30. +118 −0 src/web/SchemasPage.html
  31. +103 −0 src/web/Settings.html
  32. +103 −0 src/web/SettingsPage.html
  33. +3,932 −0 src/web/css/bootstrap.css
  34. +622 −0 src/web/css/bootstrap.min.css
  35. BIN src/web/img/glyphicons-halflings-white.png
  36. BIN src/web/img/glyphicons-halflings.png
  37. +1,734 −0 src/web/js/bootstrap.js
  38. +7 −0 src/web/js/bootstrap.min.js
View
10 bin/learnstructure
@@ -8,7 +8,9 @@ function usage {
echo "usage: learnstructure command [args...]"
echo "Commands include: "
echo " <learn> Learn the schema for a given datafile"
- echo " <analyze> Analyze a file, using schema-learning if needed."
+ echo " <analyzefile> Analyze a file, using schema-learning if needed."
+ echo " <analyzefs> Analyze a directory (or more) of files."
+ echo " <fisheye> Start the FishEye server."
exit -1
}
@@ -103,8 +105,12 @@ if [ "$CMD" = "-h" ] ; then
usage
elif [ "$CMD" = "learn" ] ; then
exec java $JOPTS $UOPTS -classpath "$CLASSPATH" com.cloudera.recordbreaker.learnstructure.LearnStructure "$@"
-elif [ "$CMD" = "analyze" ] ; then
+elif [ "$CMD" = "analyzefile" ] ; then
exec java $JOPTS $UOPTS -classpath "$CLASSPATH" com.cloudera.recordbreaker.analyzer.FormatAnalyzer "$@"
+elif [ "$CMD" = "analyzefs" ] ; then
+ exec java $JOPTS $UOPTS -classpath "$CLASSPATH" com.cloudera.recordbreaker.analyzer.FSAnalyzer "$@"
+elif [ "$CMD" = "fisheye" ] ; then
+ exec java $JOPTS $UOPTS -classpath "$CLASSPATH" com.cloudera.recordbreaker.fisheye.FishEye "$@"
elif [ "$CMD" = "class" ]; then
# Just do a java class with the environment setup
exec java $JOPTS $UOPTS -classpath "$CLASSPATH" "$@"
View
3 ivy.xml
@@ -33,6 +33,9 @@
<dependency org="org.apache.rat" name="apache-rat" rev="0.7" />
<dependency org="junit" name="junit" rev="4.8.1" />
<dependency org="net.sf.opencsv" name="opencsv" rev="2.3" />
+ <dependency org="org.mortbay.jetty" name="jetty" rev="6.1.26" />
+ <dependency org="org.apache.wicket" name="wicket-core" rev="1.5.3" />
+ <dependency org="it.tidalwave.northernwind" name="northernwind-twitter-bootstrap-1.4.0" rev="2" />
<dependency org="com.almworks.sqlite4java" name="sqlite4java" rev="0.213" />
<dependency org="com.almworks.sqlite4java" name="libsqlite4java-osx" rev="0.213">
<include type="jnilib"/>
View
36 src/java/com/cloudera/recordbreaker/analyzer/CrawlSummary.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+import java.util.*;
+
+/***********************************************************
+ * <code>CrawlSummary</code> is an interface to info about a Crawl.
+ * The data is "materialized" from the database upon the first call to
+ * one of its accessors.
+ *************************************************************/
+public class CrawlSummary {
+ FSAnalyzer analyzer;
+ long crawlid;
+
+ public CrawlSummary(FSAnalyzer analyzer, long crawlid) {
+ this.analyzer = analyzer;
+ this.crawlid = crawlid;
+ }
+
+ public String getLastExamined() {
+ return analyzer.getCrawlLastExamined(this.crawlid);
+ }
+}
View
465 src/java/com/cloudera/recordbreaker/analyzer/FSAnalyzer.java
@@ -17,12 +17,16 @@
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
+import java.util.Map;
+import java.util.TreeMap;
import java.util.List;
import java.util.Date;
import java.util.Random;
import java.util.ArrayList;
import com.almworks.sqlite4java.SQLite;
+import com.almworks.sqlite4java.SQLiteJob;
+import com.almworks.sqlite4java.SQLiteQueue;
import com.almworks.sqlite4java.SQLiteStatement;
import com.almworks.sqlite4java.SQLiteException;
import com.almworks.sqlite4java.SQLiteConnection;
@@ -49,17 +53,25 @@
static String CREATE_TABLE_FILES = "CREATE TABLE Files(fid integer primary key autoincrement, crawlid integer, fname varchar(256), owner varchar(16), size integer, modified date, path varchar(256), foreign key(crawlid) references Crawls(crawlid));";
static String CREATE_TABLE_TYPES = "CREATE TABLE Types(typeid integer primary key autoincrement, typelabel varchar(64), typedescriptor varchar(1024));";
static String CREATE_TABLE_SCHEMAS = "CREATE TABLE Schemas(schemaid integer primary key autoincrement, schemalabel varchar(64), schemadescriptor varchar(1024));";
- static String CREATE_TABLE_GUESSES = "CREATE TABLE TypeGuesses(fid integer, crawlid integer, typeid integer, schemaid integer, double score, foreign key(fid) references Files(fid), foreign key(crawlid) references Crawls(crawlid), foreign key(typeid) references Types(typeid), foreign key(schemaid) references Schemas(schemaid));";
+ static String CREATE_TABLE_GUESSES = "CREATE TABLE TypeGuesses(fid integer, typeid integer, schemaid integer, score double, foreign key(fid) references Files(fid), foreign key(typeid) references Types(typeid), foreign key(schemaid) references Schemas(schemaid));";
void createTables() throws SQLiteException {
- db.exec(CREATE_TABLE_CRAWL);
- db.exec(CREATE_TABLE_FILES);
- db.exec(CREATE_TABLE_TYPES);
- db.exec(CREATE_TABLE_SCHEMAS);
- db.exec(CREATE_TABLE_GUESSES);
+ dbQueue.execute(new SQLiteJob<Object>() {
+ protected Object job(SQLiteConnection db) throws SQLiteException {
+ try {
+ db.exec(CREATE_TABLE_CRAWL);
+ db.exec(CREATE_TABLE_FILES);
+ db.exec(CREATE_TABLE_TYPES);
+ db.exec(CREATE_TABLE_SCHEMAS);
+ db.exec(CREATE_TABLE_GUESSES);
+ } finally {
+ }
+ return null;
+ }
+ }).complete();
}
//
- // 2. Insert data
+ // 2. Insert data (about the crawled and analyzed files)
//
class TypeGuess {
String typeLabel;
@@ -96,149 +108,347 @@ String getSchemaDesc() {
* Helper fn <code>getCreateCrawl</code> returns the id of a specified Crawl in the Crawls table.
* The row is created, if necessary.
*/
- long getCreateCrawl(String lastExamined) throws SQLiteException {
- SQLiteStatement stmt = db.prepare("SELECT crawlid from Crawls WHERE lastexamined = ?");
- try {
- stmt.bind(1, lastExamined);
- if (stmt.step()) {
- long resultId = stmt.columnLong(0);
- return resultId;
- }
- } finally {
- stmt.dispose();
+ long getCreateCrawl(final String lastExamined) throws SQLiteException {
+ long crawlid = dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("SELECT crawlid from Crawls WHERE lastexamined = ?");
+ try {
+ stmt.bind(1, lastExamined);
+ if (stmt.step()) {
+ return stmt.columnLong(0);
+ } else {
+ return -1L;
+ }
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
+
+ if (crawlid >= 0) {
+ return crawlid;
}
+
// Time to insert
- stmt = db.prepare("INSERT into Crawls VALUES(null, ?)");
- try {
- stmt.bind(1, lastExamined);
- stmt.step();
- return db.getLastInsertId();
- } finally {
- stmt.dispose();
- }
+ return dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("INSERT into Crawls VALUES(null, ?)");
+ try {
+ stmt.bind(1, lastExamined);
+ stmt.step();
+ return db.getLastInsertId();
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
}
/**
* Helper fn <code>getCreateType</code> returns the id of a specified Type in the Types table.
* The row is created, if necessary.
*/
- long getCreateType(String typeLabel, String typeDesc) throws SQLiteException {
- SQLiteStatement stmt = db.prepare("SELECT typeid FROM Types WHERE typelabel = ? AND typedescriptor = ?");
- try {
- stmt.bind(1, typeLabel).bind(2, typeDesc);
- if (stmt.step()) {
- long resultId = stmt.columnLong(0);
- return resultId;
- }
- } finally {
- stmt.dispose();
+ long getCreateType(final String typeLabel, final String typeDesc) throws SQLiteException {
+ long typeid = dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("SELECT typeid FROM Types WHERE typelabel = ? AND typedescriptor = ?");
+ try {
+ stmt.bind(1, typeLabel).bind(2, typeDesc);
+ if (stmt.step()) {
+ long resultId = stmt.columnLong(0);
+ return resultId;
+ } else {
+ return -1L;
+ }
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
+
+ if (typeid >= 0) {
+ return typeid;
}
+
// Time to insert
- stmt = db.prepare("INSERT into Types VALUES(null, ?, ?)");
- try {
- stmt.bind(1, typeLabel).bind(2, typeDesc);
- stmt.step();
- return db.getLastInsertId();
- } finally {
- stmt.dispose();
- }
+ return dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("INSERT into Types VALUES(null, ?, ?)");
+ try {
+ stmt.bind(1, typeLabel).bind(2, typeDesc);
+ stmt.step();
+ return db.getLastInsertId();
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
}
/**
* Helper fn <code>getCreateSchema</code> returns the id of a specified Schema in the Schemas table.
* The row is created, if necessary.
*/
- long getCreateSchema(String schemaLabel, String schemaDesc) throws SQLiteException {
- SQLiteStatement stmt = db.prepare("SELECT schemaid FROM Schemas WHERE schemalabel = ? AND schemadescriptor = ?");
- try {
- stmt.bind(1, schemaLabel).bind(2, schemaDesc);
- if (stmt.step()) {
- long resultId = stmt.columnLong(0);
- return resultId;
- }
- } finally {
- stmt.dispose();
+ long getCreateSchema(final String schemaLabel, final String schemaDesc) throws SQLiteException {
+ long schemaid = dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ final SQLiteStatement stmt = db.prepare("SELECT schemaid FROM Schemas WHERE schemalabel = ? AND schemadescriptor = ?");
+ try {
+ stmt.bind(1, schemaLabel).bind(2, schemaDesc);
+ if (stmt.step()) {
+ long resultId = stmt.columnLong(0);
+ return resultId;
+ } else {
+ return -1L;
+ }
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
+
+ if (schemaid >= 0) {
+ return schemaid;
}
+
// Time to insert
- stmt = db.prepare("INSERT into Schemas VALUES(null, ?, ?)");
- try {
- stmt.bind(1, schemaLabel).bind(2, schemaDesc);
- stmt.step();
- return db.getLastInsertId();
- } finally {
- stmt.dispose();
- }
+ return dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ final SQLiteStatement stmt = db.prepare("INSERT into Schemas VALUES(null, ?, ?)");
+ try {
+ stmt.bind(1, schemaLabel).bind(2, schemaDesc);
+ stmt.step();
+ return db.getLastInsertId();
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
}
/**
* Add a new object to the set of all known files. This involves several tables.
*/
- long insertIntoFiles(String fname, String owner, long size, String timeDateStamp, String path, String lastExamined, List<TypeGuess> typeGuesses) throws SQLiteException {
- long crawlId = getCreateCrawl(lastExamined);
-
- long fileId = -1;
- SQLiteStatement stmt = db.prepare("INSERT into Files VALUES(null, ?, ?, ?, ?, ?, ?)");
- try {
- stmt.bind(1, crawlId).bind(2, fname).bind(3, owner).bind(4, size).bind(5, timeDateStamp).bind(6, path);
- stmt.step();
- fileId = db.getLastInsertId();
- } finally {
- stmt.dispose();
- }
+ long insertIntoFiles(final String fname, final String owner, final long size, final String timeDateStamp, final String path, String lastExamined, final List<TypeGuess> typeGuesses) throws SQLiteException {
+ final long crawlId = getCreateCrawl(lastExamined);
+
+ final long fileId = dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("INSERT into Files VALUES(null, ?, ?, ?, ?, ?, ?)");
+ try {
+ stmt.bind(1, crawlId).bind(2, fname).bind(3, owner).bind(4, size).bind(5, timeDateStamp).bind(6, path);
+ stmt.step();
+ return db.getLastInsertId();
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
// Go through all the associated typeGuesses
- for (TypeGuess tg: typeGuesses) {
- String typeLabel = tg.getTypeLabel();
- String typeDesc = tg.getTypeDesc();
- String schemaLabel = tg.getSchemaLabel();
- String schemaDesc = tg.getSchemaDesc();
- double score = tg.getScore();
-
- long typeId = getCreateType(typeLabel, typeDesc);
- long schemaId = getCreateSchema(schemaLabel, schemaDesc);
-
- stmt = db.prepare("INSERT into TypeGuesses VALUES(?, ?, ?, ?, ?)");
- try {
- stmt.bind(1, fileId).bind(2, crawlId).bind(3, typeId).bind(4, schemaId).bind(5, score);
- stmt.step();
- } finally {
- stmt.dispose();
- }
- }
+ dbQueue.execute(new SQLiteJob<Object>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ for (TypeGuess tg: typeGuesses) {
+ String typeLabel = tg.getTypeLabel();
+ String typeDesc = tg.getTypeDesc();
+ String schemaLabel = tg.getSchemaLabel();
+ String schemaDesc = tg.getSchemaDesc();
+ double score = tg.getScore();
+
+ long typeId = getCreateType(typeLabel, typeDesc);
+ long schemaId = getCreateSchema(schemaLabel, schemaDesc);
+
+ SQLiteStatement stmt = db.prepare("INSERT into TypeGuesses VALUES(?, ?, ?, ?)");
+ try {
+ stmt.bind(1, fileId).bind(2, typeId).bind(3, schemaId).bind(4, score);
+ stmt.step();
+ } finally {
+ stmt.dispose();
+ }
+ }
+ return null;
+ }
+ }).complete();
return fileId;
}
- //
- // 3. Analytics queries
- //
- // For unified file listing: select F.fname, T.typelabel, S.schemadescriptor from Schemas S, Files F, Types T, TypeGuesses TG where F.fid = TG.fid AND TG.schemaid = S.schemaid AND TG.typeid = T.typeid;
- //
- // For type statistics over entire db
- // select T.typelabel, COUNT(T.typelabel) from Files F, Types T, TypeGuesses TG where F.fid = TG.fid AND TG.typeid = T.typeid GROUP BY T.typelabel;
- //
- // For schema statistics over entire db
- // select S.schemalabel, COUNT(S.schemalabel) from Files F, Schemas S, TypeGuesses TG where F.fid = TG.fid AND TG.schemaid = S.schemaid GROUP BY S.schemalabel;
- //
- //
+ ///////////////////////////////////////////////////
+ // ACCESSORS FOR SETS OF OBJECTS
+ ///////////////////////////////////////////////////
+ /**
+ * <code>getFidUnderPath</code> returns the files under the given path prefix
+ */
static String subpathFilesQuery = "SELECT fid from Files WHERE path LIKE ?";
- List<Long> getFidUnderPath(String pathPrefix) throws SQLiteException {
- List<Long> results = new ArrayList<Long>();
- SQLiteStatement stmt = db.prepare(subpathFilesQuery);
- try {
- stmt.bind(1, pathPrefix + "%");
- while (stmt.step()) {
- long resultId = stmt.columnLong(0);
- results.add(resultId);
- }
- } finally {
- stmt.dispose();
- }
- return results;
+ public List<Long> getFidUnderPath(final String pathPrefix) throws SQLiteException {
+ List<Long> finalResults = dbQueue.execute(new SQLiteJob<List<Long>>() {
+ protected List<Long> job(SQLiteConnection db) throws SQLiteException {
+ List<Long> results = new ArrayList<Long>();
+ SQLiteStatement stmt = db.prepare(subpathFilesQuery);
+ try {
+ stmt.bind(1, pathPrefix + "%");
+ while (stmt.step()) {
+ long resultId = stmt.columnLong(0);
+ results.add(resultId);
+ }
+ return results;
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
+ return finalResults;
+ }
+
+ /**
+ * <code>getSchemaSummaries</code> returns an instance of SchemaSummary
+ * for each unique schema in the database.
+ */
+ //static String schemaInfoQuery = "SELECT Schemas.schemaid, schemalabel, schemadescriptor, Files.fid, fname FROM Schemas, Files, TypeGuesses WHERE TypeGuesses.fid = Files.fid AND TypeGuesses.schemaid = Schemas.schemaid";
+ static String schemaInfoQuery = "SELECT schemaid FROM Schemas";
+ public List<SchemaSummary> getSchemaSummaries() {
+ return dbQueue.execute(new SQLiteJob<List<SchemaSummary>>() {
+ protected List<SchemaSummary> job(SQLiteConnection db) throws SQLiteException {
+ List<SchemaSummary> output = new ArrayList<SchemaSummary>();
+ SQLiteStatement stmt = db.prepare(schemaInfoQuery);
+
+ try {
+ while (stmt.step()) {
+ long schemaId = stmt.columnLong(0);
+ output.add(new SchemaSummary(FSAnalyzer.this, schemaId));
+ }
+ } catch (SQLiteException se) {
+ se.printStackTrace();
+ } finally {
+ stmt.dispose();
+ }
+ return output;
+ }}).complete();
+ }
+
+ ///////////////////////////////////////////
+ // ACCESSORS FOR INDIVIDUAL OBJECTS
+ ///////////////////////////////////////////
+ /**
+ * Grab details on a specific file.
+ */
+ public FileSummaryData getFileSummaryData(final long fid) {
+ return dbQueue.execute(new SQLiteJob<FileSummaryData>() {
+ protected FileSummaryData job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("SELECT crawlid, fname, owner, size, modified, path from Files WHERE fid = ?");
+ try {
+ stmt.bind(1, fid);
+ if (stmt.step()) {
+ return new FileSummaryData(fid, stmt.columnLong(0), stmt.columnString(1), stmt.columnString(2), stmt.columnLong(3), stmt.columnString(4), stmt.columnString(5));
+ } else {
+ return null;
+ }
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
+ }
+
+ /**
+ * Grab details on a schema.
+ */
+ public SchemaSummaryData getSchemaSummaryData(final long schemaid) {
+ return dbQueue.execute(new SQLiteJob<SchemaSummaryData>() {
+ protected SchemaSummaryData job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("SELECT schemalabel, schemadescriptor FROM Schemas WHERE schemaid = ?");
+ try {
+ stmt.bind(1, schemaid);
+ if (stmt.step()) {
+ return new SchemaSummaryData(schemaid, stmt.columnString(0), stmt.columnString(1));
+ } else {
+ return null;
+ }
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
+ }
+
+ /**
+ * Grab details on a type.
+ */
+ public TypeSummaryData getTypeSummaryData(final long typeid) {
+ return dbQueue.execute(new SQLiteJob<TypeSummaryData>() {
+ protected TypeSummaryData job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("SELECT typelabel, typedescriptor FROM Types WHERE typeid = ?");
+ try {
+ stmt.bind(1, typeid);
+ if (stmt.step()) {
+ return new TypeSummaryData(typeid, stmt.columnString(0), stmt.columnString(1));
+ } else {
+ return null;
+ }
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
+ }
+
+ /**
+ * Grab details on a crawl.
+ */
+ public String getCrawlLastExamined(final long crawlid) {
+ return dbQueue.execute(new SQLiteJob<String>() {
+ protected String job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare("SELECT lastexamined FROM Crawls WHERE crawlid = ?");
+ try {
+ stmt.bind(1, crawlid);
+ if (stmt.step()) {
+ return stmt.columnString(0);
+ } else {
+ return null;
+ }
+ } finally {
+ stmt.dispose();
+ }
+ }
+ }).complete();
}
+
-
+ ///////////////////////////////////////////
+ // GRAB TYPE GUESS SETS
+ ///////////////////////////////////////////
+ static String typeGuessQueryForFile = "SELECT fid, typeid, schemaid, score FROM TypeGuesses WHERE fid = ?";
+ static String typeGuessQueryForSchema = "SELECT fid, typeid, schemaid, score FROM TypeGuesses WHERE schemaid = ?";
+ static String typeGuessQueryForType = "SELECT fid, typeid, schemaid, score FROM TypeGuesses WHERE typeid = ?";
+ public List<TypeGuessSummary> getTypeGuessesForFile(final long fid) {
+ return getTypeGuesses(typeGuessQueryForFile, fid);
+ }
+ public List<TypeGuessSummary> getTypeGuessesForSchema(final long schemaid) {
+ return getTypeGuesses(typeGuessQueryForSchema, schemaid);
+ }
+ public List<TypeGuessSummary> getTypeGuessesForType(final long typeid) {
+ return getTypeGuesses(typeGuessQueryForType, typeid);
+ }
+ List<TypeGuessSummary> getTypeGuesses(final String queryStr, final long idval) {
+ return dbQueue.execute(new SQLiteJob<List<TypeGuessSummary>>() {
+ protected List<TypeGuessSummary> job(SQLiteConnection db) throws SQLiteException {
+ List<TypeGuessSummary> outputList = new ArrayList<TypeGuessSummary>();
+ SQLiteStatement stmt = db.prepare(queryStr);
+ try {
+ stmt.bind(1, idval);
+ while (stmt.step()) {
+ outputList.add(new TypeGuessSummary(FSAnalyzer.this, stmt.columnLong(0), stmt.columnLong(1), stmt.columnLong(2), stmt.columnDouble(3)));
+ }
+ } finally {
+ stmt.dispose();
+ }
+ return outputList;
+ }
+ }).complete();
+ }
+
File store;
SQLiteConnection db;
+ SQLiteQueue dbQueue;
FormatAnalyzer formatAnalyzer;
/**
@@ -249,8 +459,8 @@ public FSAnalyzer(File store, File schemaDir) throws IOException, SQLiteExceptio
if (! store.exists()) {
isNew = true;
}
- this.db = new SQLiteConnection(store);
- db.open(true);
+ this.dbQueue = new SQLiteQueue(store);
+ this.dbQueue.start();
if (isNew) {
createTables();
@@ -259,8 +469,8 @@ public FSAnalyzer(File store, File schemaDir) throws IOException, SQLiteExceptio
this.formatAnalyzer = new FormatAnalyzer(schemaDir);
}
- void close() throws IOException, SQLiteException {
- db.dispose();
+ void close() throws IOException, SQLiteException, InterruptedException {
+ this.dbQueue.stop(true).join();
}
/**
@@ -317,14 +527,10 @@ void crawl(File f, int subdirDepth) throws IOException, SQLiteException {
}
}
- void query(String q) throws SQLiteException {
- SQLite.loadLibrary();
- System.err.println("SQLite version: " + SQLite.getSQLiteVersion());
- }
- public static void main(String argv[]) throws IOException, SQLiteException {
+ public static void main(String argv[]) throws Exception {
if (argv.length < 4) {
- System.err.println("Usage: FSAnalyzer <storedir> <schemaDbDir> (--crawl <dir> <subdirdepth> | --query q)");
+ System.err.println("Usage: FSAnalyzer <storedir> <schemaDbDir> (--crawl <dir> <subdirdepth>)");
return;
}
int i = 0;
@@ -338,8 +544,9 @@ public static void main(String argv[]) throws IOException, SQLiteException {
File dir = new File(argv[i++]).getCanonicalFile();
int subdirDepth = Integer.parseInt(argv[i++]);
fsa.crawl(dir, subdirDepth);
- } else if ("--query".equals(op)) {
- fsa.query(argv[i]);
+ } else if ("--test".equals(op)) {
+ List<SchemaSummary> summaryList = fsa.getSchemaSummaries();
+ System.err.println("Schema summary list has " + summaryList.size() + " entries");
}
} finally {
fsa.close();
View
84 src/java/com/cloudera/recordbreaker/analyzer/FileSummary.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+import java.util.*;
+
+/*****************************************************
+ * <code>FileSummary</code> is an interface to info about a File.
+ * The data is "materialized" from the database upon the first call to
+ * one of its accessors.
+ *****************************************************/
+public class FileSummary {
+ FSAnalyzer analyzer;
+ long fid;
+ boolean hasData = false;
+ FileSummaryData fsd = null;
+
+ public FileSummary(FSAnalyzer analyzer, long fid) {
+ this.analyzer = analyzer;
+ this.fid = fid;
+ this.hasData = false;
+ }
+
+ void getData() {
+ this.fsd = analyzer.getFileSummaryData(this.fid);
+ this.hasData = true;
+ }
+
+ public long getFid() {
+ return fid;
+ }
+
+ public String getFname() {
+ if (!hasData) {
+ getData();
+ }
+ return fsd.fname;
+ }
+ public String getOwner() {
+ if (!hasData) {
+ getData();
+ }
+ return fsd.owner;
+ }
+ public long getSize() {
+ if (!hasData) {
+ getData();
+ }
+ return fsd.size;
+ }
+ public String getLastModified() {
+ if (!hasData) {
+ getData();
+ }
+ return fsd.lastModified;
+ }
+ public String getPath() {
+ if (!hasData) {
+ getData();
+ }
+ return fsd.path;
+ }
+ public CrawlSummary getCrawl() {
+ if (!hasData) {
+ getData();
+ }
+ return new CrawlSummary(this.analyzer, fsd.crawlid);
+ }
+ public List<TypeGuessSummary> getTypeGuesses() {
+ return analyzer.getTypeGuessesForFile(this.fid);
+ }
+}
View
39 src/java/com/cloudera/recordbreaker/analyzer/FileSummaryData.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+/**
+ * The actual data for a file is contained in a <code>FileSummaryData</code> obj.
+ *
+ */
+public class FileSummaryData {
+ long fid;
+ long crawlid;
+ String fname;
+ String owner;
+ long size;
+ String lastModified;
+ String path;
+
+ public FileSummaryData(long fid, long crawlid, String fname, String owner, long size, String lastModified, String path) {
+ this.fid = fid;
+ this.crawlid = crawlid;
+ this.fname = fname;
+ this.owner = owner;
+ this.size = size;
+ this.lastModified = lastModified;
+ this.path = path;
+ }
+}
View
60 src/java/com/cloudera/recordbreaker/analyzer/SchemaSummary.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+import java.util.*;
+
+/**
+ * <code>SchemaSummary</code> captures the metadata for a schema.
+ * It has a handle to a SchemaSummaryData object, which is materialized
+ * as needed.
+ *
+ * @author "Michael Cafarella" <mjc@lofie.local>
+ * @version 1.0
+ * @since 1.0
+ */
+public class SchemaSummary {
+ FSAnalyzer analyzer;
+ long schemaId;
+ boolean hasData = false;
+ SchemaSummaryData ssd = null;
+
+ /**
+ */
+ public SchemaSummary(FSAnalyzer analyzer, long schemaId) {
+ this.analyzer = analyzer;
+ this.schemaId = schemaId;
+ this.hasData = false;
+ }
+ void getData() {
+ this.ssd = analyzer.getSchemaSummaryData(this.schemaId);
+ this.hasData = true;
+ }
+ public String getLabel() {
+ if (!hasData) {
+ getData();
+ }
+ return ssd.schemaLabel;
+ }
+ public String getDesc() {
+ if (!hasData) {
+ getData();
+ }
+ return ssd.schemaDesc;
+ }
+ public List<TypeGuessSummary> getTypeGuesses() {
+ return analyzer.getTypeGuessesForSchema(this.schemaId);
+ }
+}
View
30 src/java/com/cloudera/recordbreaker/analyzer/SchemaSummaryData.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+/**
+ * The actual data for a schema is contained in a <code>SchemaSummaryData</code> obj.
+ */
+public class SchemaSummaryData {
+ long schemaId;
+ String schemaLabel;
+ String schemaDesc;
+
+ public SchemaSummaryData(long schemaId, String schemaLabel, String schemaDesc) {
+ this.schemaId = schemaId;
+ this.schemaLabel = schemaLabel;
+ this.schemaDesc = schemaDesc;
+ }
+}
View
50 src/java/com/cloudera/recordbreaker/analyzer/TypeGuessSummary.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+import java.util.*;
+
+/************************************************************
+ * <code>TypeGuessSummary</code> is an interface to info about a type
+ * inference for a file. It is mainly a box to hold pointers to Files,
+ * Types, and Schemas.
+ *************************************************************/
+public class TypeGuessSummary {
+ FSAnalyzer analyzer;
+ long fid;
+ long typeid;
+ long schemaid;
+ double score;
+
+ public TypeGuessSummary(FSAnalyzer analyzer, long fid, long typeid, long schemaid, double score) {
+ this.analyzer = analyzer;
+ this.fid = fid;
+ this.typeid = typeid;
+ this.schemaid = schemaid;
+ this.score = score;
+ }
+ public FileSummary getFileSummary() {
+ return new FileSummary(analyzer, fid);
+ }
+ public TypeSummary getTypeSummary() {
+ return new TypeSummary(analyzer, typeid);
+ }
+ public SchemaSummary getSchemaSummary() {
+ return new SchemaSummary(analyzer, schemaid);
+ }
+ public double getScore() {
+ return score;
+ }
+}
View
56 src/java/com/cloudera/recordbreaker/analyzer/TypeSummary.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+import java.util.*;
+
+/*****************************************************
+ * <code>TypeSummary</code> is an interface to info about a file type.
+ * The data is "materialized" from the database upon the first call to
+ * one of its accessors.
+ *****************************************************/
+public class TypeSummary {
+ FSAnalyzer analyzer;
+ long typeid;
+ boolean hasData = false;
+ TypeSummaryData tsd = null;
+
+ /**
+ */
+ public TypeSummary(FSAnalyzer analyzer, long typeid) {
+ this.analyzer = analyzer;
+ this.typeid = typeid;
+ this.hasData = false;
+ }
+ void getData() {
+ this.tsd = analyzer.getTypeSummaryData(this.typeid);
+ this.hasData = true;
+ }
+ public String getLabel() {
+ if (!hasData) {
+ getData();
+ }
+ return tsd.typeLabel;
+ }
+ public String getDesc() {
+ if (!hasData) {
+ getData();
+ }
+ return tsd.typeDesc;
+ }
+ public List<TypeGuessSummary> getTypeGuesses() {
+ return analyzer.getTypeGuessesForType(this.typeid);
+ }
+}
View
30 src/java/com/cloudera/recordbreaker/analyzer/TypeSummaryData.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.analyzer;
+
+/**
+ * The actual data for a type is contained in a <code>TypeSummaryData</code> obj.
+ */
+public class TypeSummaryData {
+ long typeid;
+ String typeLabel;
+ String typeDesc;
+
+ public TypeSummaryData(long typeid, String typeLabel, String typeDesc) {
+ this.typeid = typeid;
+ this.typeLabel = typeLabel;
+ this.typeDesc = typeDesc;
+ }
+}
View
3 src/java/com/cloudera/recordbreaker/analyzer/UnknownTextDataDescriptor.java
@@ -54,9 +54,6 @@ public UnknownTextDataDescriptor(File f, File schemaDictDir) throws IOException
// 1. We already have a synthesized Avro data, with anonymous fields.
// 2. Test it against the known database of types.
// 3. Return the top-k types/schemas that we discover, as long as they pass a threshold.
- System.err.println("--------------------------------------");
- System.err.println("About to infer structure...");
- System.err.println("--------------------------------------");
LearnStructure ls = new LearnStructure();
ls.inferRecordFormat(f, workingSchemaFile, null, null, workingAvroFile, false);
View
9 src/java/com/cloudera/recordbreaker/fisheye/AboutPage.java
@@ -0,0 +1,9 @@
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+
+public class AboutPage extends WebPage {
+ public AboutPage() {
+ }
+}
View
55 src/java/com/cloudera/recordbreaker/fisheye/FilePage.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import com.cloudera.recordbreaker.analyzer.FileSummary;
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+
+/**
+ * Wicket Page class that describes a specific File
+ *
+ * @author "Michael Cafarella"
+ * @version 1.0
+ * @since 1.0
+ * @see WebPage
+ */
+public class FilePage extends WebPage {
+ public FilePage() {
+ }
+ public FilePage(PageParameters params) {
+ String fidStr = params.get("fid").toString();
+ if (fidStr != null) {
+ try {
+ FileSummary fs = new FileSummary(FishEye.analyzer, Long.parseLong(fidStr));
+ if (fs != null) {
+ add(new Label("filetitle", fs.getFname()));
+ add(new Label("filesubtitle", "in " + fs.getPath()));
+ add(new Label("owner", fs.getOwner()));
+ add(new Label("size", "" + fs.getSize()));
+ add(new Label("lastmodified", fs.getLastModified()));
+ return;
+ }
+ } catch (NumberFormatException nfe) {
+ }
+ }
+ add(new Label("filetitle", "unknown"));
+ add(new Label("filesubtitle", ""));
+ add(new Label("owner", ""));
+ add(new Label("size", ""));
+ add(new Label("lastmodified", ""));
+ }
+}
View
26 src/java/com/cloudera/recordbreaker/fisheye/FilesPage.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+
+/**
+ * The <code>FilesPage</code> renders information about all known files.
+ */
+public class FilesPage extends WebPage {
+ public FilesPage() {
+ add(new Label("numFisheyeFiles", "10"));
+ }
+}
View
31 src/java/com/cloudera/recordbreaker/fisheye/FiletypePage.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+
+/**
+ * Wicket Page class that describes a specific file type
+ *
+ * @author "Michael Cafarella"
+ * @version 1.0
+ * @since 1.0
+ * @see WebPage
+ */
+public class FiletypePage extends WebPage {
+ public FiletypePage() {
+ }
+}
View
27 src/java/com/cloudera/recordbreaker/fisheye/FiletypesPage.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+
+/**
+ * The <code>SchemasPage</code> renders information about all known filetypes
+ */
+public class FiletypesPage extends WebPage {
+ public FiletypesPage() {
+ add(new Label("numFisheyeFiletypes", "3"));
+ }
+}
View
107 src/java/com/cloudera/recordbreaker/fisheye/FishEye.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.protocol.http.WicketFilter;
+import org.apache.wicket.protocol.http.WicketServlet;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.bio.SocketConnector;
+import org.mortbay.jetty.servlet.ServletHolder;
+import org.mortbay.jetty.webapp.WebAppContext;
+import org.mortbay.resource.ResourceCollection;
+
+import com.almworks.sqlite4java.SQLiteException;
+
+import java.io.*;
+import java.util.*;
+import javax.servlet.*;
+import javax.servlet.http.*;
+
+import com.cloudera.recordbreaker.analyzer.FSAnalyzer;
+
+/***************************************************************
+ * <code>FishEye</code> is a web app that allows a user to examine
+ * the semantic contents of a filesystem. It makes extensive use of
+ * LearnStructure and other format-parsing code to help the user
+ * figure out the content of files.
+ *
+ * @author "Michael Cafarella" <mjc@cloudera.com>
+ ***************************************************************/
+public class FishEye {
+ static FSAnalyzer analyzer;
+
+ int port;
+ File htmlRoot;
+ File schemaDir;
+ File fisheyeDir;
+
+ public FishEye(int port, File htmlRoot, File schemaDir, File fisheyeDir) throws IOException, SQLiteException {
+ this.port = port;
+ this.htmlRoot = htmlRoot;
+ this.schemaDir = schemaDir;
+ this.fisheyeDir = fisheyeDir;
+ FishEye.analyzer = new FSAnalyzer(fisheyeDir, schemaDir);
+ System.err.println("Schema summary list: " + analyzer.getSchemaSummaries().size());
+ System.err.println("Schema summary list: " + analyzer.getSchemaSummaries().size());
+ }
+
+ public void run() throws Exception {
+ // Jetty object that holds the WicketServlet
+ WicketServlet ws = new WicketServlet();
+ ServletHolder servletHolder = new ServletHolder(ws);
+ servletHolder.setInitParameter("applicationClassName", "com.cloudera.recordbreaker.fisheye.FishEyeWebApplication");
+ servletHolder.setInitParameter(WicketFilter.FILTER_MAPPING_PARAM, "/*");
+ servletHolder.setInitOrder(1);
+
+ // Config the Jetty WebAppContext object
+ WebAppContext context = new WebAppContext();
+ context.addServlet(servletHolder, "/*");
+ String jarDir = this.getClass().getClassLoader().getResource("content/library/bootstrap/1.4.0").toExternalForm();
+ System.err.println("JAR URL: " + jarDir);
+ context.setBaseResource(new ResourceCollection(new String[] {htmlRoot.getCanonicalPath(), jarDir}));
+
+ // Start the HTTP server
+ Server server = new Server();
+ SocketConnector connector = new SocketConnector();
+ connector.setPort(port);
+ server.setConnectors(new Connector[]{connector});
+ server.setHandler(context);
+
+ try {
+ server.start();
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+ }
+
+ /**
+ * Start the FishEye Server.
+ */
+ public static void main(String argv[]) throws Exception {
+ if (argv.length < 4) {
+ System.err.println("Usage: FishEye <port> <htmlRoot> <schemadir> <fisheyeDir>");
+ return;
+ }
+
+ int port = Integer.parseInt(argv[0]);
+ File htmlRoot = new File(argv[1]);
+ File schemaDir = new File(argv[2]);
+ File fisheyeDir = new File(argv[3]);
+
+ FishEye fish = new FishEye(port, htmlRoot.getCanonicalFile(), schemaDir.getCanonicalFile(), fisheyeDir.getCanonicalFile());
+ fish.run();
+ }
+}
View
59 src/java/com/cloudera/recordbreaker/fisheye/FishEyeWebApplication.java
@@ -0,0 +1,59 @@
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.Page;
+import org.apache.wicket.protocol.http.WebApplication;
+import org.apache.wicket.settings.IResourceSettings;
+import org.apache.wicket.util.resource.locator.ResourceStreamLocator;
+import org.apache.wicket.util.resource.IResourceStream;
+
+
+/**
+ * The <code>FishEyeWebApplication</code> class sets up some config information
+ * for the Web app. It doesn't do a ton of interesting things.
+ *
+ */
+public class FishEyeWebApplication extends WebApplication {
+ public FishEyeWebApplication() {
+ }
+
+ class PathStripperLocator extends ResourceStreamLocator {
+ public PathStripperLocator() {
+ }
+ public IResourceStream locate(final Class clazz, final String path) {
+ IResourceStream located = super.locate(clazz, trimFolders(path));
+ if (located != null) {
+ return located;
+ }
+ return super.locate(clazz, path);
+ }
+ private String trimFolders(String path) {
+ return path.substring(path.lastIndexOf("/") + 1);
+ }
+ }
+
+ public void init() {
+ super.init();
+ IResourceSettings resourceSettings = getResourceSettings();
+ String htmlDir = this.getClass().getClassLoader().getResource("Overview.html").toExternalForm();
+ resourceSettings.addResourceFolder(htmlDir);
+ resourceSettings.setResourceStreamLocator(new PathStripperLocator());
+
+ mountPage("/Overview", Overview.class);
+ mountPage("/About", AboutPage.class);
+
+ mountPage("/Files", FilesPage.class);
+ mountPage("/Filetypes", FiletypesPage.class);
+ mountPage("/Schemas", SchemasPage.class);
+
+ mountPage("/File", FilePage.class);
+ mountPage("/Filetype", FiletypePage.class);
+ mountPage("/Schema", SchemaPage.class);
+
+ mountPage("/Settings", SettingsPage.class);
+ }
+
+ @Override
+ public Class<? extends Page> getHomePage() {
+ return Overview.class;
+ }
+}
View
10 src/java/com/cloudera/recordbreaker/fisheye/Overview.java
@@ -0,0 +1,10 @@
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+
+public class Overview extends WebPage {
+ public Overview() {
+ add(new Label("numFisheyeFiles", "10"));
+ }
+}
View
34 src/java/com/cloudera/recordbreaker/fisheye/SchemaPage.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+
+/**
+ * Wicket Page class that describes a specific Schema
+ *
+ * @author "Michael Cafarella"
+ * @version 1.0
+ * @since 1.0
+ * @see WebPage
+ */
+public class SchemaPage extends WebPage {
+ public SchemaPage() {
+ }
+ public SchemaPage(PageParameters params) {
+ }
+}
View
61 src/java/com/cloudera/recordbreaker/fisheye/SchemasPage.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.list.ListView;
+import org.apache.wicket.markup.html.list.ListItem;
+import org.apache.wicket.markup.html.link.BookmarkablePageLink;
+import org.apache.wicket.markup.html.link.ExternalLink;
+
+import java.util.List;
+import java.util.Arrays;
+
+import com.cloudera.recordbreaker.analyzer.FSAnalyzer;
+import com.cloudera.recordbreaker.analyzer.FileSummary;
+import com.cloudera.recordbreaker.analyzer.SchemaSummary;
+import com.cloudera.recordbreaker.analyzer.TypeGuessSummary;
+
+/**
+ * The <code>SchemasPage</code> renders information about all known schemas.
+ */
+public class SchemasPage extends WebPage {
+ public SchemasPage() {
+ List<SchemaSummary> list = FishEye.analyzer.getSchemaSummaries();
+ ListView<SchemaSummary> listview = new ListView<SchemaSummary>("listview", list) {
+ protected void populateItem(ListItem<SchemaSummary> item) {
+ SchemaSummary ss = item.getModelObject();
+ item.add(new Label("schemalabel", ss.getLabel()));
+ item.add(new Label("schemadesc", ss.getDesc()));
+
+ List<TypeGuessSummary> typeGuesses = ss.getTypeGuesses();
+
+ for (int i = 0; i < Math.min(1, typeGuesses.size()); i++) {
+ TypeGuessSummary curTGS = typeGuesses.get(i);
+ FileSummary fs = curTGS.getFileSummary();
+
+ PageParameters pars = new PageParameters();
+ String fidUrl = urlFor(FilePage.class, new PageParameters("fid=" + fs.getFid())).toString();
+ item.add(new ExternalLink("schemafilelink", fidUrl, fs.getFname()));
+ }
+ }
+ };
+
+ add(new Label("numFisheyeSchemas", "" + list.size()));
+ add(listview);
+ }
+}
View
31 src/java/com/cloudera/recordbreaker/fisheye/SettingsPage.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import org.apache.wicket.markup.html.WebPage;
+import org.apache.wicket.markup.html.basic.Label;
+
+/**
+ * Wicket Page class that allows user to edit Settings
+ *
+ * @author "Michael Cafarella"
+ * @version 1.0
+ * @since 1.0
+ * @see WebPage
+ */
+public class SettingsPage extends WebPage {
+ public SettingsPage() {
+ }
+}
View
84 src/web/AboutPage.html
@@ -0,0 +1,84 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Cloudera Fisheye</title>
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
+ <!--[if lt IE 9]>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+
+ <!-- Le styles -->
+ <link href="css/bootstrap.css" rel="stylesheet">
+ <style>
+ body {
+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
+ }
+ </style>
+ <link href="css/bootstrap-responsive.css" rel="stylesheet">
+
+ <!-- Le fav and touch icons -->
+ <link rel="shortcut icon" href="images/favicon.ico">
+ <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
+ <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
+ </head>
+
+ <body>
+
+ <div class="navbar navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container">
+ <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+ <a class="brand" href="/">Cloudera Fisheye</a>
+ <div class="nav-collapse">
+ <ul class="nav">
+ <li><a href="/">Home</a></li>
+ <li class="active"><a href="#">About</a></li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div class="row-fluid">
+ <div class="span12">
+ <div class="container">
+ <h1>About</h1>
+ <p>Cloudera Fisheye is a tool for understanding all of the data in your Hadoop cluster. It is built on the <a href="https://github.com/cloudera/RecordBreaker">RecordBreaker</a> project.</p>
+ <p>Fisheye v0.1 was written by Mike Cafarella.</p>
+
+ </div> <!-- /container -->
+ </div>
+ </div>
+ </div>
+
+
+
+ <!-- Le javascript
+ ================================================== -->
+ <!-- Placed at the end of the document so the pages load faster -->
+ <script src="js/jquery.js"></script>
+ <script src="js/bootstrap-transition.js"></script>
+ <script src="js/bootstrap-alert.js"></script>
+ <script src="js/bootstrap-modal.js"></script>
+ <script src="js/bootstrap-dropdown.js"></script>
+ <script src="js/bootstrap-scrollspy.js"></script>
+ <script src="js/bootstrap-tab.js"></script>
+ <script src="js/bootstrap-tooltip.js"></script>
+ <script src="js/bootstrap-popover.js"></script>
+ <script src="js/bootstrap-button.js"></script>
+ <script src="js/bootstrap-collapse.js"></script>
+ <script src="js/bootstrap-carousel.js"></script>
+ <script src="js/bootstrap-typeahead.js"></script>
+
+ </body>
+</html>
View
122 src/web/FilePage.html
@@ -0,0 +1,122 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Cloudera Fisheye</title>
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
+ <!--[if lt IE 9]>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+
+ <!-- Le styles -->
+ <link href="css/bootstrap.css" rel="stylesheet">
+ <style>
+ body {
+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
+ }
+ </style>
+ <link href="css/bootstrap-responsive.css" rel="stylesheet">
+
+ <!-- Le fav and touch icons -->
+ <link rel="shortcut icon" href="images/favicon.ico">
+ <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
+ <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
+ </head>
+
+ <body>
+
+ <div class="navbar navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container">
+ <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+ <a class="brand" href="/Overview">Cloudera Fisheye</a>
+ <div class="nav-collapse">
+ <ul class="nav">
+ <li><a href="/Overview">Home</a></li>
+ <li><a href="About">About</a></li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div class="row-fluid">
+ <div class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header"><a href="/Overview">Overview</a></li>
+ <li class="nav-header"><a href="/Files">Files</a></li>
+ <!--<li><a href="path?targetpath=asdasd">asdasd</a></li>
+ <li><a href="path?targetpath=blimfark">blimfark</a></li>
+ <li><a href="path?targetpath=foofs">foofs</a></li> -->
+ <li class="nav-header"><a href="/Filetypes">Filetypes</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Schemas">Schemas</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Settings">Settings</a></li>
+ </ul>
+ </div><!--/.well -->
+ </div>
+ <div class="span10">
+ <div class="container">
+ <h1><span wicket:id="filetitle"/></h1>
+ <h5><span wicket:id="filesubtitle"/></h5>
+
+ <table class="table table-striped table-bordered table-condensed">
+ <thead>
+ </thead>
+ <tbody>
+ <tr>
+ <td>owner</td>
+ <td><span wicket:id="owner" /></td>
+ </tr>
+ <tr>
+ <td>size</td>
+ <td><span wicket:id="size" /></td>
+ </tr>
+ <tr>
+ <td>last modified</td>
+ <td><span wicket:id="lastmodified" /></td>
+ </tr>
+ </tbody>
+ </table>
+
+ </div> <!-- /container -->
+ </div>
+ </div>
+ </div>
+
+
+
+ <!-- Le javascript
+ ================================================== -->
+ <!-- Placed at the end of the document so the pages load faster -->
+ <script src="js/jquery.js"></script>
+ <script src="js/bootstrap-transition.js"></script>
+ <script src="js/bootstrap-alert.js"></script>
+ <script src="js/bootstrap-modal.js"></script>
+ <script src="js/bootstrap-dropdown.js"></script>
+ <script src="js/bootstrap-scrollspy.js"></script>
+ <script src="js/bootstrap-tab.js"></script>
+ <script src="js/bootstrap-tooltip.js"></script>
+ <script src="js/bootstrap-popover.js"></script>
+ <script src="js/bootstrap-button.js"></script>
+ <script src="js/bootstrap-collapse.js"></script>
+ <script src="js/bootstrap-carousel.js"></script>
+ <script src="js/bootstrap-typeahead.js"></script>
+
+ </body>
+</html>
View
103 src/web/FilesPage.html
@@ -0,0 +1,103 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Cloudera Fisheye</title>
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
+ <!--[if lt IE 9]>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+
+ <!-- Le styles -->
+ <link href="css/bootstrap.css" rel="stylesheet">
+ <style>
+ body {
+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
+ }
+ </style>
+ <link href="css/bootstrap-responsive.css" rel="stylesheet">
+
+ <!-- Le fav and touch icons -->
+ <link rel="shortcut icon" href="images/favicon.ico">
+ <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
+ <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
+ </head>
+
+ <body>
+
+ <div class="navbar navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container">
+ <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+ <a class="brand" href="/Overview">Cloudera Fisheye</a>
+ <div class="nav-collapse">
+ <ul class="nav">
+ <li><a href="/Overview">Home</a></li>
+ <li><a href="About">About</a></li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div class="row-fluid">
+ <div class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header"><a href="/Overview">Overview</a></li>
+ <li class="nav-header active"><a href="/Files">Files</a></li>
+ <!--<li><a href="path?targetpath=asdasd">asdasd</a></li>
+ <li><a href="path?targetpath=blimfark">blimfark</a></li>
+ <li><a href="path?targetpath=foofs">foofs</a></li> -->
+ <li class="nav-header"><a href="/Filetypes">Filetypes</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Schemas">Schemas</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Settings">Settings</a></li>
+ </ul>
+ </div><!--/.well -->
+ </div>
+ <div class="span10">
+ <div class="container">
+ <h1>Files</h1>
+ <p>Fisheye is currently staring at <span wicket:id="numFisheyeFiles"></span> files.</p>
+
+ </div> <!-- /container -->
+ </div>
+ </div>
+ </div>
+
+
+
+ <!-- Le javascript
+ ================================================== -->
+ <!-- Placed at the end of the document so the pages load faster -->
+ <script src="js/jquery.js"></script>
+ <script src="js/bootstrap-transition.js"></script>
+ <script src="js/bootstrap-alert.js"></script>
+ <script src="js/bootstrap-modal.js"></script>
+ <script src="js/bootstrap-dropdown.js"></script>
+ <script src="js/bootstrap-scrollspy.js"></script>
+ <script src="js/bootstrap-tab.js"></script>
+ <script src="js/bootstrap-tooltip.js"></script>
+ <script src="js/bootstrap-popover.js"></script>
+ <script src="js/bootstrap-button.js"></script>
+ <script src="js/bootstrap-collapse.js"></script>
+ <script src="js/bootstrap-carousel.js"></script>
+ <script src="js/bootstrap-typeahead.js"></script>
+
+ </body>
+</html>
View
103 src/web/FiletypesPage.html
@@ -0,0 +1,103 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Cloudera Fisheye</title>
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
+ <!--[if lt IE 9]>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+
+ <!-- Le styles -->
+ <link href="css/bootstrap.css" rel="stylesheet">
+ <style>
+ body {
+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
+ }
+ </style>
+ <link href="css/bootstrap-responsive.css" rel="stylesheet">
+
+ <!-- Le fav and touch icons -->
+ <link rel="shortcut icon" href="images/favicon.ico">
+ <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
+ <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
+ </head>
+
+ <body>
+
+ <div class="navbar navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container">
+ <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+ <a class="brand" href="/Overview">Cloudera Fisheye</a>
+ <div class="nav-collapse">
+ <ul class="nav">
+ <li><a href="/Overview">Home</a></li>
+ <li><a href="About">About</a></li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div class="row-fluid">
+ <div class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header"><a href="/Overview">Overview</a></li>
+ <li class="nav-header"><a href="/Files">Files</a></li>
+ <!--<li><a href="path?targetpath=asdasd">asdasd</a></li>
+ <li><a href="path?targetpath=blimfark">blimfark</a></li>
+ <li><a href="path?targetpath=foofs">foofs</a></li> -->
+ <li class="nav-header active"><a href="/Filetypes">Filetypes</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Schemas">Schemas</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Settings">Settings</a></li>
+ </ul>
+ </div><!--/.well -->
+ </div>
+ <div class="span10">
+ <div class="container">
+ <h1>Filetypes</h1>
+ <p>Fisheye is currently gazing at <span wicket:id="numFisheyeFiletypes"></span> filetypes.</p>
+
+ </div> <!-- /container -->
+ </div>
+ </div>
+ </div>
+
+
+
+ <!-- Le javascript
+ ================================================== -->
+ <!-- Placed at the end of the document so the pages load faster -->
+ <script src="js/jquery.js"></script>
+ <script src="js/bootstrap-transition.js"></script>
+ <script src="js/bootstrap-alert.js"></script>
+ <script src="js/bootstrap-modal.js"></script>
+ <script src="js/bootstrap-dropdown.js"></script>
+ <script src="js/bootstrap-scrollspy.js"></script>
+ <script src="js/bootstrap-tab.js"></script>
+ <script src="js/bootstrap-tooltip.js"></script>
+ <script src="js/bootstrap-popover.js"></script>
+ <script src="js/bootstrap-button.js"></script>
+ <script src="js/bootstrap-collapse.js"></script>
+ <script src="js/bootstrap-carousel.js"></script>
+ <script src="js/bootstrap-typeahead.js"></script>
+
+ </body>
+</html>
View
103 src/web/Overview.html
@@ -0,0 +1,103 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Cloudera Fisheye</title>
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
+ <!--[if lt IE 9]>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+
+ <!-- Le styles -->
+ <link href="css/bootstrap.css" rel="stylesheet">
+ <style>
+ body {
+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
+ }
+ </style>
+ <link href="css/bootstrap-responsive.css" rel="stylesheet">
+
+ <!-- Le fav and touch icons -->
+ <link rel="shortcut icon" href="images/favicon.ico">
+ <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
+ <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
+ </head>
+
+ <body>
+
+ <div class="navbar navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container">
+ <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+ <a class="brand" href="/Overview">Cloudera Fisheye</a>
+ <div class="nav-collapse">
+ <ul class="nav">
+ <li class="active"><a href="/Overview">Home</a></li>
+ <li><a href="About">About</a></li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div class="row-fluid">
+ <div class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header active"><a href="/Overview">Overview</a></li>
+ <li class="nav-header"><a href="/Files">Files</a></li>
+ <!--<li><a href="path?targetpath=asdasd">asdasd</a></li>
+ <li><a href="path?targetpath=blimfark">blimfark</a></li>
+ <li><a href="path?targetpath=foofs">foofs</a></li> -->
+ <li class="nav-header"><a href="/Filetypes">Filetypes</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Schemas">Schemas</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Settings">Settings</a></li>
+ </ul>
+ </div><!--/.well -->
+ </div>
+ <div class="span10">
+ <div class="container">
+ <h1>Overview</h1>
+ <p>Fisheye is currently staring at <span wicket:id="numFisheyeFiles"></span> files.</p>
+
+ </div> <!-- /container -->
+ </div>
+ </div>
+ </div>
+
+
+
+ <!-- Le javascript
+ ================================================== -->
+ <!-- Placed at the end of the document so the pages load faster -->
+ <script src="js/jquery.js"></script>
+ <script src="js/bootstrap-transition.js"></script>
+ <script src="js/bootstrap-alert.js"></script>
+ <script src="js/bootstrap-modal.js"></script>
+ <script src="js/bootstrap-dropdown.js"></script>
+ <script src="js/bootstrap-scrollspy.js"></script>
+ <script src="js/bootstrap-tab.js"></script>
+ <script src="js/bootstrap-tooltip.js"></script>
+ <script src="js/bootstrap-popover.js"></script>
+ <script src="js/bootstrap-button.js"></script>
+ <script src="js/bootstrap-collapse.js"></script>
+ <script src="js/bootstrap-carousel.js"></script>
+ <script src="js/bootstrap-typeahead.js"></script>
+
+ </body>
+</html>
View
103 src/web/SchemaPage.html
@@ -0,0 +1,103 @@
+<!DOCTYPE html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Cloudera Fisheye</title>
+ <meta name="description" content="">
+ <meta name="author" content="">
+
+ <!-- Le HTML5 shim, for IE6-8 support of HTML elements -->
+ <!--[if lt IE 9]>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+
+ <!-- Le styles -->
+ <link href="css/bootstrap.css" rel="stylesheet">
+ <style>
+ body {
+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
+ }
+ </style>
+ <link href="css/bootstrap-responsive.css" rel="stylesheet">
+
+ <!-- Le fav and touch icons -->
+ <link rel="shortcut icon" href="images/favicon.ico">
+ <link rel="apple-touch-icon" href="images/apple-touch-icon.png">
+ <link rel="apple-touch-icon" sizes="72x72" href="images/apple-touch-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="images/apple-touch-icon-114x114.png">
+ </head>
+
+ <body>
+
+ <div class="navbar navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container">
+ <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+ <a class="brand" href="/Overview">Cloudera Fisheye</a>
+ <div class="nav-collapse">
+ <ul class="nav">
+ <li><a href="/Overview">Home</a></li>
+ <li><a href="About">About</a></li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+ </div>
+ </div>
+
+ <div class="container-fluid">
+ <div class="row-fluid">
+ <div class="span2">
+ <div class="well sidebar-nav">
+ <ul class="nav nav-list">
+ <li class="nav-header"><a href="/Overview">Overview</a></li>
+ <li class="nav-header"><a href="/Files">Files</a></li>
+ <!--<li><a href="path?targetpath=asdasd">asdasd</a></li>
+ <li><a href="path?targetpath=blimfark">blimfark</a></li>
+ <li><a href="path?targetpath=foofs">foofs</a></li> -->
+ <li class="nav-header"><a href="/Filetypes">Filetypes</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Schemas">Schemas</a></li>
+ <!-- <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li>
+ <li><a href="#">Link</a></li> -->
+ <li class="nav-header"><a href="/Settings">Settings</a></li>
+ </ul>
+ </div><!--/.well -->
+ </div>
+ <div class="span10">
+ <div class="container">
+ <h1>Schema</h1>
+ <p>Schema.</p>
+
+ </div> <!-- /container -->
+ </div>
+ </div>
+ </div>
+
+
+
+ <!-- Le javascript
+ ================================================== -->
+ <!-- Placed at the end of the document so the pages load faster -->
+ <script src="js/jquery.js"></script>
+ <script src="js/bootstrap-transition.js"></script>
+ <script src="js/bootstrap-alert.js"></script>
+ <script src="js/bootstrap-modal.js"></script>
+ <script src="js/bootstrap-dropdown.js"></script>
+ <script src="js/bootstrap-scrollspy.js"></script>
+ <script src="js/bootstrap-tab.js"></script>
+ <script src="js/bootstrap-tooltip.js"></script>
+ <script src="js/bootstrap-popover.js"></script>
+ <script src="js/bootstrap-button.js"></script>
+ <script src="js/bootstrap-collapse.js"></script>
+ <script src="js/bootstrap-carousel.js"></script>
+ <script src="js/bootstrap-typeahead.js"></script>
+
+ </body>
+</html>
View
118 src/web/SchemasPage.html