Permalink
Browse files

Merging changes r1036303:r1036692 from trunk to federation

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/branches/HDFS-1052@1078873 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent d089784 commit 4ee28a0b354aaeb27dbe2487644b0d240360a66e Suresh Srinivas committed Mar 7, 2011
View
@@ -18,6 +18,10 @@
.project
.launches/
.settings
+*.iml
+*.ipr
+*.iws
+.idea
.svn
build/
build-fi/
View
@@ -298,6 +298,9 @@ Release 0.22.0 - Unreleased
HDFS-895. Allow hflush/sync to occur in parallel with new writes
to the file. (Todd Lipcon via hairong)
+ HDFS-528. Add ability for safemode to wait for a minimum number of
+ live datanodes (Todd Lipcon via eli)
+
IMPROVEMENTS
HDFS-1304. Add a new unit test for HftpFileSystem.open(..). (szetszwo)
@@ -625,6 +628,11 @@ Release 0.22.0 - Unreleased
HDFS-874. TestHDFSFileContextMainOperations fails on weirdly
configured DNS hosts. (Todd Lipcon via eli)
+ HDFS-1507. TestAbandonBlock should abandon a block. (eli)
+
+ HDFS-1487. FSDirectory.removeBlock() should update diskspace count
+ of the block owner node (Zhong Wang via eli).
+
Release 0.21.1 - Unreleased
HDFS-1411. Correct backup node startup command in hdfs user guide.
@@ -344,12 +344,27 @@ creations/deletions), or "all".</description>
<description>
Specifies the percentage of blocks that should satisfy
the minimal replication requirement defined by dfs.namenode.replication.min.
- Values less than or equal to 0 mean not to start in safe mode.
+ Values less than or equal to 0 mean not to wait for any particular
+ percentage of blocks before exiting safemode.
Values greater than 1 will make safe mode permanent.
</description>
</property>
<property>
+ <name>dfs.namenode.safemode.min.datanodes</name>
+ <value>0</value>
+ <description>
+ Specifies the number of datanodes that must be considered alive
+ before the name node exits safemode.
+ Values less than or equal to 0 mean not to take the number of live
+ datanodes into account when deciding whether to remain in safe mode
+ during startup.
+ Values greater than the number of datanodes in the cluster
+ will make safe mode permanent.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.safemode.extension</name>
<value>30000</value>
<description>
@@ -58,6 +58,8 @@
public static final int DFS_NAMENODE_SAFEMODE_EXTENSION_DEFAULT = 30000;
public static final String DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY = "dfs.namenode.safemode.threshold-pct";
public static final float DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_DEFAULT = 0.999f;
+ public static final String DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY = "dfs.namenode.safemode.min.datanodes";
+ public static final int DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT = 0;
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY = "dfs.namenode.secondary.http-address";
public static final String DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_DEFAULT = "0.0.0.0:50090";
public static final String DFS_NAMENODE_CHECKPOINT_PERIOD_KEY = "dfs.namenode.checkpoint.period";
@@ -453,6 +453,11 @@ boolean removeBlock(String path, INodeFileUnderConstruction fileNode,
+path+" with "+block
+" block is added to the file system");
}
+
+ // update space consumed
+ INode[] pathINodes = getExistingPathINodes(path);
+ updateCount(pathINodes, pathINodes.length-1, 0,
+ -fileNode.getPreferredBlockSize()*fileNode.getReplication(), true);
} finally {
writeUnlock();
}
@@ -2638,6 +2638,10 @@ nodes with its data cleared (or user can just remove the StorageID
// no need to update its timestamp
// because its is done when the descriptor is created
}
+
+ if (safeMode != null) {
+ safeMode.checkMode();
+ }
return;
} finally {
writeUnlock();
@@ -2996,6 +3000,10 @@ private void removeDatanode(DatanodeDescriptor nodeInfo) {
}
unprotectedRemoveDatanode(nodeInfo);
clusterMap.remove(nodeInfo);
+
+ if (safeMode != null) {
+ safeMode.checkMode();
+ }
}
void unprotectedRemoveDatanode(DatanodeDescriptor nodeDescr) {
@@ -3778,6 +3786,8 @@ DatanodeDescriptor getRandomDatanode() {
// configuration fields
/** Safe mode threshold condition %.*/
private double threshold;
+ /** Safe mode minimum number of datanodes alive */
+ private int datanodeThreshold;
/** Safe mode extension after the threshold. */
private int extension;
/** Min replication required by safe mode. */
@@ -3807,6 +3817,9 @@ DatanodeDescriptor getRandomDatanode() {
*/
SafeModeInfo(Configuration conf) {
this.threshold = conf.getFloat(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY, 0.95f);
+ this.datanodeThreshold = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY,
+ DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_DEFAULT);
this.extension = conf.getInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0);
this.safeReplication = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
@@ -3824,6 +3837,7 @@ DatanodeDescriptor getRandomDatanode() {
*/
private SafeModeInfo() {
this.threshold = 1.5f; // this threshold can never be reached
+ this.datanodeThreshold = Integer.MAX_VALUE;
this.extension = Integer.MAX_VALUE;
this.safeReplication = Short.MAX_VALUE + 1; // more than maxReplication
this.blockTotal = -1;
@@ -3916,7 +3930,8 @@ synchronized boolean canLeave() {
* if DFS is empty or {@link #threshold} == 0
*/
boolean needEnter() {
- return threshold != 0 && blockSafe < blockThreshold;
+ return (threshold != 0 && blockSafe < blockThreshold) ||
+ (getNumLiveDataNodes() < datanodeThreshold);
}
/**
@@ -4006,17 +4021,37 @@ String getTurnOffTip() {
}
if(blockTotal < 0)
return leaveMsg + ".";
-
- String msg = null;
+
+ int numLive = getNumLiveDataNodes();
+ String msg = "";
if (reached == 0) {
- msg = String.format("The reported blocks %d needs additional %d"
- + " blocks to reach the threshold %.4f of total blocks %d. %s",
- blockSafe, (blockThreshold - blockSafe), threshold, blockTotal,
- leaveMsg);
+ if (blockSafe < blockThreshold) {
+ msg += String.format(
+ "The reported blocks %d needs additional %d"
+ + " blocks to reach the threshold %.4f of total blocks %d.",
+ blockSafe, (blockThreshold - blockSafe), threshold, blockTotal);
+ }
+ if (numLive < datanodeThreshold) {
+ if (!"".equals(msg)) {
+ msg += "\n";
+ }
+ msg += String.format(
+ "The number of live datanodes %d needs an additional %d live "
+ + "datanodes to reach the minimum number %d.",
+ numLive, datanodeThreshold - numLive, datanodeThreshold);
+ }
+ msg += " " + leaveMsg;
} else {
msg = String.format("The reported blocks %d has reached the threshold"
- + " %.4f of total blocks %d. %s", blockSafe, threshold,
- blockTotal, leaveMsg);
+ + " %.4f of total blocks %d.", blockSafe, threshold,
+ blockTotal);
+
+ if (datanodeThreshold > 0) {
+ msg += String.format(" The number of live datanodes %d has reached "
+ + "the minimum number %d.",
+ numLive, datanodeThreshold);
+ }
+ msg += " " + leaveMsg;
}
if(reached == 0 || isManual()) { // threshold is not reached or manual
return msg + ".";
@@ -23,49 +23,86 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
-import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-public class TestAbandonBlock extends junit.framework.TestCase {
+import static org.junit.Assert.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test abandoning blocks, which clients do on pipeline creation failure.
+ */
+public class TestAbandonBlock {
public static final Log LOG = LogFactory.getLog(TestAbandonBlock.class);
private static final Configuration CONF = new HdfsConfiguration();
static final String FILE_NAME_PREFIX
= "/" + TestAbandonBlock.class.getSimpleName() + "_";
+ private MiniDFSCluster cluster;
+ private FileSystem fs;
- public void testAbandonBlock() throws IOException {
- MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(2).build();
- FileSystem fs = cluster.getFileSystem();
+ @Before
+ public void setUp() throws Exception {
+ cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(2).build();
+ fs = cluster.getFileSystem();
+ cluster.waitActive();
+ }
+ @After
+ public void tearDown() throws Exception {
+ fs.close();
+ cluster.shutdown();
+ }
+
+ @Test
+ /** Abandon a block while creating a file */
+ public void testAbandonBlock() throws IOException {
String src = FILE_NAME_PREFIX + "foo";
- FSDataOutputStream fout = null;
- try {
- //start writing a a file but not close it
- fout = fs.create(new Path(src), true, 4096, (short)1, 512L);
- for(int i = 0; i < 1024; i++) {
- fout.write(123);
- }
- fout.hflush();
-
- //try reading the block by someone
- final DFSClient dfsclient = new DFSClient(NameNode.getAddress(CONF), CONF);
- LocatedBlocks blocks = dfsclient.getNamenode().getBlockLocations(src, 0, 1);
- LocatedBlock b = blocks.get(0);
- try {
- dfsclient.getNamenode().abandonBlock(b.getBlock(), src, "someone");
- //previous line should throw an exception.
- assertTrue(false);
- }
- catch(IOException ioe) {
- LOG.info("GREAT! " + StringUtils.stringifyException(ioe));
- }
+
+ // Start writing a file but do not close it
+ FSDataOutputStream fout = fs.create(new Path(src), true, 4096, (short)1, 512L);
+ for (int i = 0; i < 1024; i++) {
+ fout.write(123);
+ }
+ fout.hflush();
+
+ // Now abandon the last block
+ DFSClient dfsclient = ((DistributedFileSystem)fs).getClient();
+ LocatedBlocks blocks = dfsclient.getNamenode().getBlockLocations(src, 0, 1);
+ LocatedBlock b = blocks.getLastLocatedBlock();
+ dfsclient.getNamenode().abandonBlock(b.getBlock(), src, dfsclient.clientName);
+
+ // And close the file
+ fout.close();
+ }
+
+ @Test
+ /** Make sure that the quota is decremented correctly when a block is abandoned */
+ public void testQuotaUpdatedWhenBlockAbandoned() throws IOException {
+ DistributedFileSystem dfs = (DistributedFileSystem)fs;
+ // Setting diskspace quota to 3MB
+ dfs.setQuota(new Path("/"), FSConstants.QUOTA_DONT_SET, 3 * 1024 * 1024);
+
+ // Start writing a file with 2 replicas to ensure each datanode has one.
+ // Block Size is 1MB.
+ String src = FILE_NAME_PREFIX + "test_quota1";
+ FSDataOutputStream fout = fs.create(new Path(src), true, 4096, (short)2, 1024 * 1024);
+ for (int i = 0; i < 1024; i++) {
+ fout.writeByte(123);
}
- finally {
- try{fout.close();} catch(Exception e) {}
- try{fs.close();} catch(Exception e) {}
- try{cluster.shutdown();} catch(Exception e) {}
+
+ // Shutdown one datanode, causing the block abandonment.
+ cluster.getDataNodes().get(0).shutdown();
+
+ // Close the file, new block will be allocated with 2MB pending size.
+ try {
+ fout.close();
+ } catch (QuotaExceededException e) {
+ fail("Unexpected quota exception when closing fout");
}
}
}
Oops, something went wrong.

0 comments on commit 4ee28a0

Please sign in to comment.