Skip to content
This repository has been archived by the owner on Jan 13, 2022. It is now read-only.

Commit

Permalink
[Raid Dir-Raid] Web UI changes for directory raid
Browse files Browse the repository at this point in the history 
Summary:
For directory raid, we may be more interested in the number of RAIDED,
NOT_RAIDED_TOO_NEW, NOT_RAIDED_TOO_SMALL, NOT_RAIDED_BUT_SHOULD directories
than that of files.

Change the webui to adopt the codec design and counters to include the counter of
directory.

The counters will be updated by the StatisticsCollector, which is another task.

Test Plan: deploy

Reviewers: dikang, hkuang

Reviewed By: hkuang

Task ID: 1078275
  • Loading branch information
weiyan authored and Alex Feinberg committed Jul 7, 2012
1 parent f6505cf commit a91ff6f
Show file tree
Hide file tree
Showing 4 changed files with 67 additions and 61 deletions.
13 changes: 7 additions & 6 deletions src/contrib/raid/src/java/org/apache/hadoop/raid/Codec.java
View file
Expand Up @@ -19,6 +19,7 @@
package org.apache.hadoop.raid;

import java.io.IOException;
import java.io.Serializable;
import java.lang.IllegalArgumentException;
import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -44,7 +45,7 @@
* 3. Parity directory location
* 4. Codec priority
*/
public class Codec {
public class Codec implements Serializable {

public static final Log LOG = LogFactory.getLog(Codec.class);

Expand All @@ -53,7 +54,7 @@ public class Codec {
/**
* Used by ErasureCode.init() to get Code specific extra parameters.
*/
public final JSONObject json;
public final String jsonStr;

/**
* id of the codec. Used by policy in raid.xml
Expand Down Expand Up @@ -177,7 +178,7 @@ public int compare(Codec c1, Codec c2) {
}

private Codec(JSONObject json) throws JSONException {
this.json = json;
this.jsonStr = json.toString();
this.id = json.getString("id");
this.parityLength = json.getInt("parity_length");
this.stripeLength = json.getInt("stripe_length");
Expand Down Expand Up @@ -235,10 +236,10 @@ public ErasureCode createErasureCode(Configuration conf) {

@Override
public String toString() {
if (json == null) {
if (jsonStr == null) {
return "Test codec " + id;
} else {
return json.toString();
return jsonStr;
}
}

Expand Down Expand Up @@ -287,7 +288,7 @@ static void clearCodecs() {
String tmpHarDirectory,
boolean isDirRaid,
boolean simulateBlockFix) {
this.json = null;
this.jsonStr = null;
this.id = id;
this.parityLength = parityLength;
this.stripeLength = stripeLength;
Expand Down
43 changes: 33 additions & 10 deletions src/contrib/raid/src/java/org/apache/hadoop/raid/Statistics.java
View file
Expand Up @@ -21,6 +21,7 @@
import java.io.Serializable;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
Expand All @@ -35,6 +36,7 @@
public class Statistics implements Serializable {

final private String codecId;
final private Codec codec;
final private int parityLength;
final private int stripeLength;
private long estimatedParitySize = 0L;
Expand All @@ -46,6 +48,7 @@ public class Statistics implements Serializable {
private Map<Integer, Counters> numBlocksToRaidedCounters;

public Statistics(Codec codec, Configuration conf) {
this.codec = codec;
this.codecId = codec.id;
this.stripeLength = codec.stripeLength;
this.parityLength = codec.parityLength;
Expand All @@ -59,6 +62,7 @@ public Statistics(Codec codec, Configuration conf) {
}

public static class Counters implements Serializable {
private long numDirs = 0L;
private long numFiles = 0L;
private long numBlocks = 0L;
private long numBytes = 0L;
Expand All @@ -70,6 +74,19 @@ private void inc(FileStatus status) {
numLogical += status.getLen();
numBytes += status.getLen() * status.getReplication();
}

/**
* Increment counters for directory
* @param status file status of directory
* @param lfs List of FileStatus for files under the direcotry
*/
private void inc(FileStatus status, List<FileStatus> lfs) {
//TODO
}

public long getNumDirs() {
return numDirs;
}

public long getNumFiles() {
return numFiles;
Expand All @@ -92,14 +109,16 @@ public boolean equals(Object obj) {
return false;
}
Counters counters = (Counters) obj;
return (numFiles == counters.numFiles &&
return (numDirs == counters.numDirs &&
numFiles == counters.numFiles &&
numBlocks == counters.numBlocks &&
numBytes == counters.numBytes &&
numLogical == counters.numLogical);
}
@Override
public int hashCode() {
int hash = 7;
hash = 37 * hash + (int) (numDirs ^ (numDirs >>> 32));
hash = 37 * hash + (int) (numFiles ^ (numFiles >>> 32));
hash = 37 * hash + (int) (numBlocks ^ (numBlocks >>> 32));
hash = 37 * hash + (int) (numBytes ^ (numBytes >>> 32));
Expand All @@ -108,17 +127,21 @@ public int hashCode() {
}
@Override
public String toString() {
return "files:" + numFiles + " blocks:" + numBlocks +
" bytes:" + numBytes + " logical:" + numLogical;
return "dirs: " + numDirs + "files:" + numFiles + " blocks:"
+ numBlocks + " bytes:" + numBytes + " logical:" + numLogical;
}
public String htmlRow() {
return td(StringUtils.humanReadableInt(numFiles)) +
public String htmlRow(Codec codec) {
String dirColumn = !codec.isDirRaid? "":
td(StringUtils.humanReadableInt(numDirs));
return dirColumn + td(StringUtils.humanReadableInt(numFiles)) +
td(StringUtils.humanReadableInt(numBlocks)) +
td(StringUtils.byteDesc(numBytes)) +
td(StringUtils.byteDesc(numLogical));
}
public static String htmlRowHeader() {
return td("Files") + td("Blocks") +
public static String htmlRowHeader(Codec codec) {
String dirHeader = !codec.isDirRaid? "":
td("Dirs");
return dirHeader + td("Files") + td("Blocks") +
td("Bytes") + td("Logical");
}
}
Expand Down Expand Up @@ -296,12 +319,12 @@ public String htmlTable() {
RaidState.NOT_RAIDED_TOO_SMALL, RaidState.NOT_RAIDED_BUT_SHOULD};

StringBuilder sb = new StringBuilder();
sb.append(tr(td("STATE") + Counters.htmlRowHeader()));
sb.append(tr(td("STATE") + Counters.htmlRowHeader(codec)));
for (RaidState state : statesToShow) {
Counters counters = stateToSourceCounters.get(state);
sb.append(tr(td(state.toString()) + counters.htmlRow()));
sb.append(tr(td(state.toString()) + counters.htmlRow(codec)));
}
sb.append(tr(td("PARITY") + parityCounters.htmlRow()));
sb.append(tr(td("PARITY") + parityCounters.htmlRow(codec)));
return table(sb.toString());
}

Expand Down
2 changes: 1 addition & 1 deletion src/contrib/raid/src/test/org/apache/hadoop/raid/TestStatisticsCollector.java
View file
Expand Up @@ -50,7 +50,6 @@ public class TestStatisticsCollector extends TestCase {
final FakeConfigManager fakeConfigManager = new FakeConfigManager();
final FakeRaidNode fakeRaidNode = new FakeRaidNode();


public void testExcludes() throws IOException {
conf.set("raid.exclude.patterns", "/exclude/,/df_mf/");
RaidState.Checker checker = new RaidState.Checker(
Expand Down Expand Up @@ -106,6 +105,7 @@ public void testSnapshot() throws Exception {
"/tmp/raidStatsSnapshot");
MiniDFSCluster dfs = null;
try {
Utils.loadTestCodecs(conf);
dfs = new MiniDFSCluster(conf, 3, true, null);
dfs.waitActive();
FileSystem fs = dfs.getFileSystem();
Expand Down
70 changes: 26 additions & 44 deletions src/contrib/raid/webapps/raid/raid.jsp
View file
Expand Up @@ -16,8 +16,6 @@
RaidNode raidNode = (RaidNode) application.getAttribute("raidnode");
StatisticsCollector stats = (StatisticsCollector) raidNode
.getStatsCollector();
Statistics xorSt = stats.getRaidStatistics("xor");
Statistics rsSt = stats.getRaidStatistics("rs");
PurgeMonitor purge = raidNode.getPurgeMonitor();
PlacementMonitor place = raidNode.getPlacementMonitor();
DiskStatus ds = new DFSClient(raidNode.getConf()).getNSDiskStatus();
Expand Down Expand Up @@ -85,51 +83,35 @@
out.print(table(tableStr));
%>
<hr>
<h2>XOR</h2>
<%
String paritySize, estParitySize;
if (xorSt != null) {
out.print(xorSt.htmlTable());
saving = StringUtils.byteDesc(xorSt.getSaving());
doneSaving = StringUtils.byteDesc(xorSt.getDoneSaving());
repl = StringUtils.limitDecimalTo2(xorSt.getEffectiveReplication());
paritySize = StringUtils.byteDesc(xorSt.getParityCounters()
.getNumBytes());
estParitySize = StringUtils.byteDesc(xorSt.getEstimatedParitySize());
tableStr = "";
tableStr += tr(td("Effective Replication") + td(":") + td(repl));
tableStr += tr(td("Saving") + td(":") + td(saving));
tableStr += tr(td("Done Saving") + td(":") + td(doneSaving));
tableStr += tr(td("Parity / Expected") + td(":")
+ td(paritySize + " / " + estParitySize));
out.print(table(tableStr));
} else {
out.print("Wait for collecting");
}
%>
<hr>
<h2>RS</h2>
<%
if (rsSt != null) {
out.print(rsSt.htmlTable());
saving = StringUtils.byteDesc(rsSt.getSaving());
doneSaving = StringUtils.byteDesc(rsSt.getDoneSaving());
repl = StringUtils.limitDecimalTo2(rsSt.getEffectiveReplication());
paritySize = StringUtils.byteDesc(rsSt.getParityCounters()
.getNumBytes());
estParitySize = StringUtils.byteDesc(rsSt.getEstimatedParitySize());
tableStr = "";
tableStr += tr(td("Effective Replication") + td(":") + td(repl));
tableStr += tr(td("Saving") + td(":") + td(saving));
tableStr += tr(td("Done Saving") + td(":") + td(doneSaving));
tableStr += tr(td("Parity / Expected") + td(":")
+ td(paritySize + " / " + estParitySize));
out.print(table(tableStr));
} else {
out.print("Wait for collecting");
for (Codec codec: Codec.getCodecs()) {
out.print("\n<h2>" + codec.id + " (" +
(codec.isDirRaid?"Directory-level":
"File-level") +
") " + "</h2>\n");
Statistics codeStats = stats.getRaidStatistics(codec.id);
String paritySize, estParitySize;
if (codeStats != null) {
out.print(codeStats.htmlTable());
saving = StringUtils.byteDesc(codeStats.getSaving());
doneSaving = StringUtils.byteDesc(codeStats.getDoneSaving());
repl = StringUtils.limitDecimalTo2(codeStats.getEffectiveReplication());
paritySize = StringUtils.byteDesc(codeStats.getParityCounters()
.getNumBytes());
estParitySize = StringUtils.byteDesc(codeStats.getEstimatedParitySize());
tableStr = "";
tableStr += tr(td("Effective Replication") + td(":") + td(repl));
tableStr += tr(td("Saving") + td(":") + td(saving));
tableStr += tr(td("Done Saving") + td(":") + td(doneSaving));
tableStr += tr(td("Parity / Expected") + td(":")
+ td(paritySize + " / " + estParitySize));
out.print(table(tableStr));
} else {
out.print("Wait for collecting");
}
out.print("\n<hr>\n");
}
%>
<hr>
<h2>Purge Progress</h2>
<%
out.print(purge.htmlTable());
Expand Down

0 comments on commit a91ff6f

Please sign in to comment.