Skip to content

Commit

Permalink
HADOOP-7713. dfs -count -q should label output column (Jonathan Allen…
Browse files Browse the repository at this point in the history 
… via aw)
  • Loading branch information
@aw-was-here
aw-was-here committed Feb 5, 2015
1 parent 4e7ad4f commit fb06c00
Show file tree
Hide file tree
Showing 8 changed files with 237 additions and 46 deletions.
3 changes: 3 additions & 0 deletions hadoop-common-project/hadoop-common/CHANGES.txt
View file
Expand Up @@ -154,6 +154,9 @@ Trunk (Unreleased)
HADOOP-10976. moving the source code of hadoop-tools docs to the
directory under hadoop-tools (Masatake Iwasaki via aw)

HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
via aw)

BUG FIXES

HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
Expand Down
57 changes: 41 additions & 16 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
View file
Expand Up @@ -97,28 +97,35 @@ public void readFields(DataInput in) throws IOException {
this.spaceConsumed = in.readLong();
this.spaceQuota = in.readLong();
}
/**

/**
* Output format:
* <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
private static final String STRING_FORMAT = "%12s %12s %18s ";
/**
private static final String SUMMARY_FORMAT = "%12s %12s %18s ";
/**
* Output format:
* <----12----> <----15----> <----15----> <----15----> <----12----> <----12----> <-------18------->
* QUOTA REMAINING_QUATA SPACE_QUOTA SPACE_QUOTA_REM DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME
* <----12----> <------15-----> <------15-----> <------15----->
* QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA
* <----12----> <----12----> <-------18------->
* DIR_COUNT FILE_COUNT CONTENT_SIZE
*/
private static final String QUOTA_STRING_FORMAT = "%12s %15s ";
private static final String SPACE_QUOTA_STRING_FORMAT = "%15s %15s ";

private static final String QUOTA_SUMMARY_FORMAT = "%12s %15s ";
private static final String SPACE_QUOTA_SUMMARY_FORMAT = "%15s %15s ";

private static final String[] HEADER_FIELDS = new String[] { "DIR_COUNT",
"FILE_COUNT", "CONTENT_SIZE"};
private static final String[] QUOTA_HEADER_FIELDS = new String[] { "QUOTA",
"REM_QUOTA", "SPACE_QUOTA", "REM_SPACE_QUOTA" };

/** The header string */
private static final String HEADER = String.format(
STRING_FORMAT.replace('d', 's'), "directories", "files", "bytes");
SUMMARY_FORMAT, (Object[]) HEADER_FIELDS);

private static final String QUOTA_HEADER = String.format(
QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
"name quota", "rem name quota", "space quota", "rem space quota") +
QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
(Object[]) QUOTA_HEADER_FIELDS) +
HEADER;

/** Return the header of the output.
Expand All @@ -131,7 +138,25 @@ public void readFields(DataInput in) throws IOException {
public static String getHeader(boolean qOption) {
return qOption ? QUOTA_HEADER : HEADER;
}


/**
* Returns the names of the fields from the summary header.
*
* @return names of fields as displayed in the header
*/
public static String[] getHeaderFields() {
return HEADER_FIELDS;
}

/**
* Returns the names of the fields used in the quota summary.
*
* @return names of quota fields as displayed in the header
*/
public static String[] getQuotaHeaderFields() {
return QUOTA_HEADER_FIELDS;
}

@Override
public String toString() {
return toString(true);
Expand Down Expand Up @@ -175,11 +200,11 @@ public String toString(boolean qOption, boolean hOption) {
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}

prefix = String.format(QUOTA_STRING_FORMAT + SPACE_QUOTA_STRING_FORMAT,
prefix = String.format(QUOTA_SUMMARY_FORMAT + SPACE_QUOTA_SUMMARY_FORMAT,
quotaStr, quotaRem, spaceQuotaStr, spaceQuotaRem);
}

return prefix + String.format(STRING_FORMAT,
return prefix + String.format(SUMMARY_FORMAT,
formatSize(directoryCount, hOption),
formatSize(fileCount, hOption),
formatSize(length, hOption));
Expand Down
32 changes: 22 additions & 10 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
View file
Expand Up @@ -21,6 +21,7 @@
import java.util.Arrays;
import java.util.LinkedList;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
Expand All @@ -44,18 +45,26 @@ public static void registerCommands(CommandFactory factory) {

private static final String OPTION_QUOTA = "q";
private static final String OPTION_HUMAN = "h";
private static final String OPTION_HEADER = "v";

public static final String NAME = "count";
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] <path> ...";
public static final String DESCRIPTION =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] <path> ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
"that match the specified file pattern. The output columns are:\n" +
"DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or\n" +
"QUOTA REMAINING_QUOTA SPACE_QUOTA REMAINING_SPACE_QUOTA \n" +
" DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME\n" +
"The -h option shows file sizes in human readable format.";

"that match the specified file pattern. The output columns are:\n" +
StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
" PATHNAME\n" +
"or, with the -" + OPTION_QUOTA + " option:\n" +
StringUtils.join(ContentSummary.getQuotaHeaderFields(), ' ') + "\n" +
" " +
StringUtils.join(ContentSummary.getHeaderFields(), ' ') +
" PATHNAME\n" +
"The -" + OPTION_HUMAN +
" option shows file sizes in human readable format.\n" +
"The -" + OPTION_HEADER + " option displays a header line.";

private boolean showQuotas;
private boolean humanReadable;

Expand All @@ -65,7 +74,7 @@ public Count() {}
/** Constructor
* @deprecated invoke via {@link FsShell}
* @param cmd the count command
* @param pos the starting index of the arguments
* @param pos the starting index of the arguments
* @param conf configuration
*/
@Deprecated
Expand All @@ -77,13 +86,16 @@ public Count(String[] cmd, int pos, Configuration conf) {
@Override
protected void processOptions(LinkedList<String> args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN);
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
args.add(".");
}
showQuotas = cf.getOpt(OPTION_QUOTA);
humanReadable = cf.getOpt(OPTION_HUMAN);
if (cf.getOpt(OPTION_HEADER)) {
out.println(ContentSummary.getHeader(showQuotas) + "PATHNAME");
}
}

@Override
Expand Down
12 changes: 8 additions & 4 deletions hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
View file
Expand Up @@ -154,17 +154,19 @@ bin/hadoop fs <args>

* count

Usage: <<<hadoop fs -count [-q] [-h] <paths> >>>
Usage: <<<hadoop fs -count [-q] [-h] [-v] <paths> >>>

Count the number of directories, files and bytes under the paths that match
the specified file pattern. The output columns with -count are: DIR_COUNT,
FILE_COUNT, CONTENT_SIZE FILE_NAME
FILE_COUNT, CONTENT_SIZE PATHNAME

The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA,
REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, FILE_NAME
REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME

The -h option shows sizes in human readable format.

The -v option displays a header line.

Example:

* <<<hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
Expand All @@ -173,6 +175,8 @@ bin/hadoop fs <args>

* <<<hadoop fs -count -q -h hdfs://nn1.example.com/file1>>>

* <<<hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1>>>

Exit Code:

Returns 0 on success and -1 on error.
Expand Down Expand Up @@ -743,4 +747,4 @@ permissions userid groupid modification_date modification_time dirname

Usage: <<<hadoop fs -usage command>>>

Return the help for an individual command.
Return the help for an individual command.
6 changes: 3 additions & 3 deletions ...p-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java
View file
Expand Up @@ -137,15 +137,15 @@ public void testReadFields() throws IOException {
// check the header with quotas
@Test
public void testGetHeaderWithQuota() {
String header = " name quota rem name quota space quota "
+ "rem space quota directories files bytes ";
String header = " QUOTA REM_QUOTA SPACE_QUOTA "
+ "REM_SPACE_QUOTA DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(true));
}

// check the header without quotas
@Test
public void testGetHeaderNoQuota() {
String header = " directories files bytes ";
String header = " DIR_COUNT FILE_COUNT CONTENT_SIZE ";
assertEquals(header, ContentSummary.getHeader(false));
}

Expand Down
115 changes: 106 additions & 9 deletions hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java
View file
Expand Up @@ -31,6 +31,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.CommandFormat.NotEnoughArgumentsException;
import org.junit.Test;
import org.junit.Before;
import org.junit.BeforeClass;
Expand All @@ -47,15 +48,13 @@ public class TestCount {
private static Configuration conf;
private static FileSystem mockFs;
private static FileStatus fileStat;
private static ContentSummary mockCs;

@BeforeClass
public static void setup() {
conf = new Configuration();
conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
mockFs = mock(FileSystem.class);
fileStat = mock(FileStatus.class);
mockCs = mock(ContentSummary.class);
when(fileStat.isFile()).thenReturn(true);
}

Expand Down Expand Up @@ -87,6 +86,85 @@ public void processOptionsAll() {
assertTrue(count.isHumanReadable());
}

// check no options is handled correctly
@Test
public void processOptionsNoOptions() {
LinkedList<String> options = new LinkedList<String>();
options.add("dummy");
Count count = new Count();
count.processOptions(options);
assertFalse(count.isShowQuotas());
}

// check -q is handled correctly
@Test
public void processOptionsShowQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-q");
options.add("dummy");
Count count = new Count();
count.processOptions(options);
assertTrue(count.isShowQuotas());
}

// check missing arguments is handled correctly
@Test
public void processOptionsMissingArgs() {
LinkedList<String> options = new LinkedList<String>();
Count count = new Count();
try {
count.processOptions(options);
fail("Count.processOptions - NotEnoughArgumentsException not thrown");
} catch (NotEnoughArgumentsException e) {
}
assertFalse(count.isShowQuotas());
}

// check the correct header is produced with no quotas (-v)
@Test
public void processOptionsHeaderNoQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-v");
options.add("dummy");

PrintStream out = mock(PrintStream.class);

Count count = new Count();
count.out = out;

count.processOptions(options);

String noQuotasHeader =
// <----12----> <----12----> <-------18------->
" DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
verify(out).println(noQuotasHeader);
verifyNoMoreInteractions(out);
}

// check the correct header is produced with quotas (-q -v)
@Test
public void processOptionsHeaderWithQuotas() {
LinkedList<String> options = new LinkedList<String>();
options.add("-q");
options.add("-v");
options.add("dummy");

PrintStream out = mock(PrintStream.class);

Count count = new Count();
count.out = out;

count.processOptions(options);

String withQuotasHeader =
// <----12----> <-----15------> <-----15------> <-----15------>
" QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA " +
// <----12----> <----12----> <-------18------->
" DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME";
verify(out).println(withQuotasHeader);
verifyNoMoreInteractions(out);
}

// check quotas are reported correctly
@Test
public void processPathShowQuotas() throws Exception {
Expand Down Expand Up @@ -211,29 +289,48 @@ public void getName() {
public void getUsage() {
Count count = new Count();
String actual = count.getUsage();
String expected = "-count [-q] [-h] <path> ...";
String expected = "-count [-q] [-h] [-v] <path> ...";
assertEquals("Count.getUsage", expected, actual);
}

// check the correct description is returned
@Test
public void getDescription() {
Count count = new Count();
String actual = count.getDescription();
String expected =
"Count the number of directories, files and bytes under the paths\n"
+ "that match the specified file pattern. The output columns are:\n"
+ "DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "or, with the -q option:\n"
+ "QUOTA REM_QUOTA SPACE_QUOTA REM_SPACE_QUOTA\n"
+ " DIR_COUNT FILE_COUNT CONTENT_SIZE PATHNAME\n"
+ "The -h option shows file sizes in human readable format.\n"
+ "The -v option displays a header line.";

assertEquals("Count.getDescription", expected, actual);
}


// mock content system
static class MockContentSummary extends ContentSummary {

public MockContentSummary() {}

public MockContentSummary() {
}

@Override
public String toString(boolean qOption, boolean hOption) {
if (qOption) {
if (hOption) {
return(HUMAN + WITH_QUOTAS);
return (HUMAN + WITH_QUOTAS);
} else {
return(BYTES + WITH_QUOTAS);
return (BYTES + WITH_QUOTAS);
}
} else {
if (hOption) {
return(HUMAN + NO_QUOTAS);
return (HUMAN + NO_QUOTAS);
} else {
return(BYTES + NO_QUOTAS);
return (BYTES + NO_QUOTAS);
}
}
}
Expand Down

0 comments on commit fb06c00

Please sign in to comment.