Permalink
Browse files

HDFS-1295. Port to yahoo-merge branch. (mattf)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hdfs/branches/yahoo-merge@1134449 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
mattf-horton committed Jun 10, 2011
1 parent 8f3a282 commit 97f02dbcdf202979987cbb9b27bdd46d0e068094
View
@@ -378,6 +378,9 @@ Trunk (unreleased changes)
HDFS-1905. Improve namenode -format command by not making -clusterId
parameter mandatory. (Bharath Mundlapudi via suresh)
+ HDFS-1295. Improve namenode restart times by short-circuiting the
+ first block reports from datanodes. (Matt Foley via suresh)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
@@ -909,27 +909,30 @@ void notifyNamenodeReceivedBlock(ExtendedBlock block, String delHint) {
* @throws IOException
*/
DatanodeCommand blockReport() throws IOException {
- // send block report
+ // send block report if timer has expired.
DatanodeCommand cmd = null;
long startTime = now();
if (startTime - lastBlockReport > blockReportInterval) {
- //
- // Send latest block report if timer has expired.
- // Get back a list of local block(s) that are obsolete
- // and can be safely GC'ed.
- //
- long brStartTime = now();
+
+ // Create block report
+ long brCreateStartTime = now();
BlockListAsLongs bReport = data.getBlockReport(blockPoolId);
+
+ // Send block report
+ long brSendStartTime = now();
cmd = bpNamenode.blockReport(bpRegistration, blockPoolId, bReport
.getBlockListAsLongs());
- long brTime = now() - brStartTime;
- metrics.addBlockReport(brTime);
- LOG.info("BlockReport of " + bReport.getNumberOfBlocks() +
- " blocks got processed in " + brTime + " msecs");
- //
+
+ // Log the block report processing stats from Datanode perspective
+ long brSendCost = now() - brSendStartTime;
+ long brCreateCost = brSendStartTime - brCreateStartTime;
+ metrics.addBlockReport(brSendCost);
+ LOG.info("BlockReport of " + bReport.getNumberOfBlocks()
+ + " blocks took " + brCreateCost + " msec to generate and "
+ + brSendCost + " msecs for RPC and NN processing");
+
// If we have sent the first block report, then wait a random
// time before we start the periodic block reports.
- //
if (resetBlockReportTime) {
lastBlockReport = startTime - R.nextInt((int)(blockReportInterval));
resetBlockReportTime = false;
Oops, something went wrong.

0 comments on commit 97f02db

Please sign in to comment.