Skip to content
Browse files

Improve Schema page, rationalize type 'identifier' and 'description' …

…fields. Check in AccessControlWarningBox
  • Loading branch information...
1 parent 530a658 commit ac889eefe734174f542a1108f95a47aa7a5c3841 @mikecafarella mikecafarella committed
View
35 src/java/com/cloudera/recordbreaker/analyzer/FSAnalyzer.java
@@ -59,7 +59,7 @@
static String CREATE_TABLE_FILESYSTEM = "CREATE TABLE Filesystems(fsid integer primary key autoincrement, fsname text);";
static String CREATE_TABLE_FILES = "CREATE TABLE Files(fid integer primary key autoincrement, isDir string, crawlid integer, fname varchar(256), owner varchar(16), groupowner varchar(16), permissions varchar(32), size integer, modified date, path varchar(256), foreign key(crawlid) references Crawls(crawlid));";
static String CREATE_TABLE_TYPES = "CREATE TABLE Types(typeid integer primary key autoincrement, typelabel varchar(64), typedescriptor varchar(1024));";
- static String CREATE_TABLE_SCHEMAS = "CREATE TABLE Schemas(schemaid integer primary key autoincrement, schemalabel varchar(64), schemadescriptor varchar(1024));";
+ static String CREATE_TABLE_SCHEMAS = "CREATE TABLE Schemas(schemaid integer primary key autoincrement, schemaidentifier varchar(1024), schemadescription varchar(32));";
static String CREATE_TABLE_GUESSES = "CREATE TABLE TypeGuesses(fid integer, typeid integer, schemaid integer, score double, foreign key(fid) references Files(fid), foreign key(typeid) references Types(typeid), foreign key(schemaid) references Schemas(schemaid));";
void createTables() throws SQLiteException {
dbQueue.execute(new SQLiteJob<Object>() {
@@ -259,12 +259,12 @@ protected Long job(SQLiteConnection db) throws SQLiteException {
* Helper fn <code>getCreateSchema</code> returns the id of a specified Schema in the Schemas table.
* The row is created, if necessary.
*/
- long getCreateSchema(final String schemaLabel, final String schemaDesc) throws SQLiteException {
+ long getCreateSchema(final String schemaIdentifier, final String schemaDesc) throws SQLiteException {
long schemaid = dbQueue.execute(new SQLiteJob<Long>() {
protected Long job(SQLiteConnection db) throws SQLiteException {
- final SQLiteStatement stmt = db.prepare("SELECT schemaid FROM Schemas WHERE schemalabel = ? AND schemadescriptor = ?");
+ final SQLiteStatement stmt = db.prepare("SELECT schemaid FROM Schemas WHERE schemaidentifier = ? AND schemadescription = ?");
try {
- stmt.bind(1, schemaLabel).bind(2, schemaDesc);
+ stmt.bind(1, schemaIdentifier).bind(2, schemaDesc);
if (stmt.step()) {
long resultId = stmt.columnLong(0);
return resultId;
@@ -286,7 +286,7 @@ protected Long job(SQLiteConnection db) throws SQLiteException {
protected Long job(SQLiteConnection db) throws SQLiteException {
final SQLiteStatement stmt = db.prepare("INSERT into Schemas VALUES(null, ?, ?)");
try {
- stmt.bind(1, schemaLabel).bind(2, schemaDesc);
+ stmt.bind(1, schemaIdentifier).bind(2, schemaDesc);
stmt.step();
return db.getLastInsertId();
} finally {
@@ -346,10 +346,10 @@ protected Long job(SQLiteConnection db) throws SQLiteException {
for (TypeGuess tg: typeGuesses) {
String typeLabel = tg.getTypeLabel();
String typeDesc = tg.getTypeDesc();
- String schemaLabel = tg.getSchemaLabel();
+ String schemaIdentifier = tg.getSchemaIdentifier();
String schemaDesc = tg.getSchemaDesc();
typeIds.add(getCreateType(typeLabel, typeDesc));
- schemaIds.add(getCreateSchema(schemaLabel, schemaDesc));
+ schemaIds.add(getCreateSchema(schemaIdentifier, schemaDesc));
}
dbQueue.execute(new SQLiteJob<Object>() {
protected Long job(SQLiteConnection db) throws SQLiteException {
@@ -414,7 +414,7 @@ DataDescriptor describeData(FileSystem fs, Path p, int maxLines) throws IOExcept
public SchemaSummaryData getSchemaSummaryData(final long schemaid) {
return dbQueue.execute(new SQLiteJob<SchemaSummaryData>() {
protected SchemaSummaryData job(SQLiteConnection db) throws SQLiteException {
- SQLiteStatement stmt = db.prepare("SELECT schemalabel, schemadescriptor FROM Schemas WHERE schemaid = ?");
+ SQLiteStatement stmt = db.prepare("SELECT schemaidentifier, schemadescription FROM Schemas WHERE schemaid = ?");
try {
stmt.bind(1, schemaid);
if (stmt.step()) {
@@ -803,13 +803,30 @@ protected Object job(SQLiteConnection db) throws SQLiteException {
///////////////////////////////////////////
static String typeGuessQueryForFile = "SELECT fid, typeid, schemaid, score FROM TypeGuesses WHERE fid = ?";
static String typeGuessQueryForSchema = "SELECT fid, typeid, schemaid, score FROM TypeGuesses WHERE schemaid = ?";
- static String typeGuessQueryForType = "SELECT fid, typeid, schemaid, score FROM TypeGuesses WHERE typeid = ?";
+ static String typeGuessQueryForType = "SELECT fid, typeid, schemaid, score FROM TypeGuesses WHERE typeid = ?";
public List<TypeGuessSummary> getTypeGuessesForFile(final long fid) {
return getTypeGuesses(typeGuessQueryForFile, fid);
}
public List<TypeGuessSummary> getTypeGuessesForSchema(final long schemaid) {
return getTypeGuesses(typeGuessQueryForSchema, schemaid);
}
+ static String countFilesQueryForSchema = "SELECT COUNT(DISTINCT fid) FROM TypeGuesses WHERE schemaid = ?";
+ public long countFilesForSchema(final long schemaid) {
+ return dbQueue.execute(new SQLiteJob<Long>() {
+ protected Long job(SQLiteConnection db) throws SQLiteException {
+ SQLiteStatement stmt = db.prepare(countFilesQueryForSchema);
+ try {
+ stmt.bind(1, schemaid);
+ if (stmt.step()) {
+ return stmt.columnLong(0);
+ }
+ } finally {
+ stmt.dispose();
+ }
+ return -1L;
+ }
+ }).complete();
+ }
public List<TypeGuessSummary> getTypeGuessesForType(final long typeid) {
return getTypeGuesses(typeGuessQueryForType, typeid);
}
View
2 src/java/com/cloudera/recordbreaker/analyzer/FSCrawler.java
@@ -75,7 +75,7 @@ protected synchronized void addSingleFile(FileSystem fs, Path p, long crawlid) t
if (schemas == null || schemas.size() == 0) {
tgs.add(new TypeGuess(descriptor.getFileTypeIdentifier(), descriptor.getFileTypeIdentifier(),
- "no schema", "no schema", 1.0));
+ "", "no schema", 1.0));
} else {
for (SchemaDescriptor sd: schemas) {
tgs.add(new TypeGuess(descriptor.getFileTypeIdentifier(), descriptor.getFileTypeIdentifier(),
View
4 src/java/com/cloudera/recordbreaker/analyzer/SchemaSummary.java
@@ -45,11 +45,11 @@ void getData() {
this.ssd = analyzer.getSchemaSummaryData(this.schemaId);
this.hasData = true;
}
- public String getLabel() {
+ public String getIdentifier() {
if (!hasData) {
getData();
}
- return ssd.schemaLabel;
+ return ssd.schemaIdentifier;
}
public String getDesc() {
if (!hasData) {
View
6 src/java/com/cloudera/recordbreaker/analyzer/SchemaSummaryData.java
@@ -19,12 +19,12 @@
*/
public class SchemaSummaryData {
long schemaId;
- String schemaLabel;
+ String schemaIdentifier;
String schemaDesc;
- public SchemaSummaryData(long schemaId, String schemaLabel, String schemaDesc) {
+ public SchemaSummaryData(long schemaId, String schemaIdentifier, String schemaDesc) {
this.schemaId = schemaId;
- this.schemaLabel = schemaLabel;
+ this.schemaIdentifier = schemaIdentifier;
this.schemaDesc = schemaDesc;
}
}
View
10 src/java/com/cloudera/recordbreaker/analyzer/TypeGuess.java
@@ -25,14 +25,14 @@
public class TypeGuess {
String typeLabel;
String typeDesc;
- String schemaLabel;
+ String schemaIdentifier;
String schemaDesc;
double score;
- public TypeGuess(String typeLabel, String typeDesc, String schemaLabel, String schemaDesc, double score) {
+ public TypeGuess(String typeLabel, String typeDesc, String schemaIdentifier, String schemaDesc, double score) {
this.typeLabel = typeLabel;
this.typeDesc = typeDesc;
- this.schemaLabel = schemaLabel;
+ this.schemaIdentifier = schemaIdentifier;
this.schemaDesc = schemaDesc;
this.score = score;
}
@@ -42,8 +42,8 @@ String getTypeLabel() {
String getTypeDesc() {
return typeDesc;
}
- String getSchemaLabel() {
- return schemaLabel;
+ String getSchemaIdentifier() {
+ return schemaIdentifier;
}
String getSchemaDesc() {
return schemaDesc;
View
67 src/java/com/cloudera/recordbreaker/fisheye/AccessControlWarningBox.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2012, Cloudera, Inc. All Rights Reserved.
+ *
+ * Cloudera, Inc. licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"). You may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This software is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License for
+ * the specific language governing permissions and limitations under the
+ * License.
+ */
+package com.cloudera.recordbreaker.fisheye;
+
+import com.cloudera.recordbreaker.analyzer.FSAnalyzer;
+import com.cloudera.recordbreaker.analyzer.FileSummary;
+
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.WebMarkupContainer;
+
+/************************************************
+ * Application-wide warning box that tells user
+ * when she does not have access for the current file/dir
+ *
+ * @author "Michael Cafarella"
+ * @version 1.0
+ * @since 1.0
+ *************************************************/
+public class AccessControlWarningBox extends WebMarkupContainer {
+ String targetName = null;
+ int targetFid = -1;
+
+ public AccessControlWarningBox(String name, int targetFid) {
+ super(name);
+ this.targetFid = targetFid;
+ setOutputMarkupPlaceholderTag(true);
+ setVisibilityAllowed(false);
+ }
+
+ public AccessControlWarningBox(String name, String targetName) {
+ super(name);
+ this.targetName = targetName;
+ setOutputMarkupPlaceholderTag(true);
+ setVisibilityAllowed(false);
+ }
+
+ public void onConfigure() {
+ FishEye fe = FishEye.getInstance();
+ AccessController accessCtrl = fe.getAccessController();
+ FSAnalyzer fsAnalyzer = fe.getAnalyzer();
+
+ FileSummary fileSummary = null;
+ if (targetFid >= 0) {
+ fileSummary = new FileSummary(fsAnalyzer, targetFid);
+ } else if (targetName != null) {
+ fileSummary = fsAnalyzer.getSingleFileSummary(targetName);
+ }
+
+ if (fileSummary != null) {
+ setVisibilityAllowed(fe.hasFSAndCrawl() && !accessCtrl.hasReadAccess(fileSummary));
+ } else {
+ setVisibilityAllowed(false);
+ }
+ }
+}
View
7 src/java/com/cloudera/recordbreaker/fisheye/FiletypePage.java
@@ -54,8 +54,11 @@ public FiletypeDisplay(String name, String filetypeStr) {
List<TypeGuessSummary> tgses = ts.getTypeGuesses();
add(new Label("typecount", "" + tgses.size()));
-
- // Files that have this type
+ //
+ // REMIND -- mjc -- I think this file-listing display is more trouble than its worth, at least
+ // when examining filesystems of non-trivial size. But I'm not 100% sure yet, so it will stick
+ // around a bit...
+ //
/**
ListView<TypeGuessSummary> observationList = new ListView<TypeGuessSummary>("observations", tgses) {
protected void populateItem(ListItem<TypeGuessSummary> item) {
View
18 src/java/com/cloudera/recordbreaker/fisheye/SchemaPage.java
@@ -89,12 +89,23 @@ public SchemaPageDisplay(String name, String schemaidStr) {
if (fe.hasFSAndCrawl()) {
List<List<JsonNode>> listOfSchemaElts = new ArrayList<List<JsonNode>>();
+ long numFilesWithSchema = -1;
+ String schemaDescription = "";
if (schemaidStr != null) {
try {
- SchemaSummary ss = new SchemaSummary(fe.getAnalyzer(), Long.parseLong(schemaidStr));
- listOfSchemaElts = getSchemaDigest(ss.getLabel());
+ long schemaId = Long.parseLong(schemaidStr);
+ SchemaSummary ss = new SchemaSummary(fe.getAnalyzer(), schemaId);
+ if (ss.getIdentifier().length() > 0) {
+ listOfSchemaElts = getSchemaDigest(ss.getIdentifier());
+ } else {
+ listOfSchemaElts = new ArrayList<List<JsonNode>>();
+ }
+ schemaDescription = ss.getDesc();
+ numFilesWithSchema = fe.getAnalyzer().countFilesForSchema(schemaId);
} catch (NumberFormatException nfe) {
+ nfe.printStackTrace();
} catch (IOException ie) {
+ ie.printStackTrace();
}
}
@@ -114,7 +125,10 @@ protected void populateItem(ListItem<JsonNode> item2) {
item.add(listview);
}
});
+
+ add(new Label("schemaDesc", schemaDescription));
add(new Label("numSchemaElements", "" + listOfSchemaElts.size()));
+ add(new Label("numFilesWithSchema", "" + numFilesWithSchema));
}
setOutputMarkupPlaceholderTag(true);
View
2 src/java/com/cloudera/recordbreaker/fisheye/SchemasPage.java
@@ -52,7 +52,7 @@ protected void populateItem(ListItem<SchemaSummary> item) {
StringBuffer schemalabel = new StringBuffer();
try {
- List<List<JsonNode>> listOfSchemaElts = SchemaPage.getSchemaDigest(ss.getLabel());
+ List<List<JsonNode>> listOfSchemaElts = SchemaPage.getSchemaDigest(ss.getIdentifier());
for (Iterator<JsonNode> it = listOfSchemaElts.get(0).iterator(); it.hasNext(); ) {
JsonNode curNode = it.next();
schemalabel.append(curNode.get("name"));
View
3 src/web/fisheye/SchemaPage.html
@@ -86,8 +86,9 @@
</div>
<div wicket:id="currentSchemaDisplay">
- <p>Fisheye is currently boring into <span wicket:id="numSchemaElements"></span> schema(s).</p>
+ <h2>of type '<span wicket:id="schemaDesc"/>'</h2>
+ <p>This schema has <span wicket:id="numSchemaElements"></span> field(s). <span wicket:id="numFilesWithSchema"></span> file(s) have this schema.</p>
<table>
<tr wicket:id="biglistview">
<table class="table table-striped table-bordered table-condensed">

0 comments on commit ac889ee

Please sign in to comment.
Something went wrong with that request. Please try again.