Skip to content

Commit

Permalink
HDFS-12105. Ozone: listVolumes doesn't work from ozone commandline. C…
Browse files Browse the repository at this point in the history 
…ontributed by Yiqun Lin.
  • Loading branch information
@yangwwei @omalley
yangwwei authored and omalley committed Apr 26, 2018
1 parent 7afddca commit ea5e24c
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 24 deletions.
60 changes: 45 additions & 15 deletions ...project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
View file
Expand Up @@ -204,24 +204,26 @@ public OzoneVolume getVolume(String volumeName) throws OzoneException {
* List all the volumes owned by the user or Owned by the user specified in
* the behalf of string.
*
* @param onBehalfOf - User Name of the user if it is not the caller. for
* example, an admin wants to list some other users
* volumes.
* @param prefix - Return only volumes that match this prefix.
* @param maxKeys - Maximum number of results to return, if the result set
* is smaller than requested size, it means that list is
* complete.
* @param prevKey - The last key that client got, server will continue
* returning results from that point.
* @param onBehalfOf
* User Name of the user if it is not the caller. for example,
* an admin wants to list some other users volumes.
* @param prefix
* Return only volumes that match this prefix.
* @param maxKeys
* Maximum number of results to return, if the result set
* is smaller than requested size, it means that list is
* complete.
* @param startVolume
* The previous volume name.
* @return List of Volumes
* @throws OzoneException
*/
public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
maxKeys, OzoneVolume prevKey) throws OzoneException {
public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
int maxKeys, String startVolume) throws OzoneException {
HttpGet httpGet = null;
try (CloseableHttpClient httpClient = newHttpClient()) {
URIBuilder builder = new URIBuilder(endPointURI);
if (prefix != null) {
if (!Strings.isNullOrEmpty(prefix)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
}

Expand All @@ -230,9 +232,9 @@ public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
.toString(maxKeys));
}

if (prevKey != null) {
if (!Strings.isNullOrEmpty(startVolume)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY,
prevKey.getOwnerName() + "/" + prevKey.getVolumeName());
startVolume);
}

builder.setPath("/").build();
Expand All @@ -249,6 +251,33 @@ public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
}
}

/**
* List all the volumes owned by the user or Owned by the user specified in
* the behalf of string.
*
* @param onBehalfOf - User Name of the user if it is not the caller. for
* example, an admin wants to list some other users
* volumes.
* @param prefix - Return only volumes that match this prefix.
* @param maxKeys - Maximum number of results to return, if the result set
* is smaller than requested size, it means that list is
* complete.
* @param prevKey - The last key that client got, server will continue
* returning results from that point.
* @return List of Volumes
* @throws OzoneException
*/
public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
int maxKeys, OzoneVolume prevKey) throws OzoneException {
String volumeName = null;

if (prevKey != null) {
volumeName = prevKey.getVolumeName();
}

return listVolumes(onBehalfOf, prefix, maxKeys, volumeName);
}

/**
* List volumes of the current user or if onBehalfof is not null lists volume
* owned by that user. You need admin privilege to read other users volume
Expand All @@ -260,7 +289,8 @@ public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix, int
*/
public List<OzoneVolume> listVolumes(String onBehalfOf)
throws OzoneException {
return listVolumes(onBehalfOf, null, 1000, null);
return listVolumes(onBehalfOf, null,
Integer.parseInt(Header.OZONE_DEFAULT_LIST_SIZE), StringUtils.EMPTY);
}

/**
Expand Down
34 changes: 28 additions & 6 deletions ...doop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
View file
Expand Up @@ -55,18 +55,39 @@ protected void execute(CommandLine cmd)
"Incorrect call : listVolume is missing");
}

int maxKeys = 0;
if (cmd.hasOption(Shell.LIST_LENGTH)) {
String length = cmd.getOptionValue(Shell.LIST_LENGTH);
try {
maxKeys = Integer.parseInt(length);
} catch (NumberFormatException nfe) {
throw new OzoneRestClientException(
"Invalid max key length, the vaule should be digital.");
}

if (maxKeys <= 0) {
throw new OzoneRestClientException(
"Invalid max key length, the vaule should be a positive number.");
}
}

String startVolume = null;
if (cmd.hasOption(Shell.START)) {
startVolume = cmd.getOptionValue(Shell.START);
}

String prefix = null;
if (cmd.hasOption(Shell.PREFIX)) {
prefix = cmd.getOptionValue(Shell.PREFIX);
}

String ozoneURIString = cmd.getOptionValue(Shell.LIST_VOLUME);
URI ozoneURI = verifyURI(ozoneURIString);

if (cmd.hasOption(Shell.RUNAS)) {
rootName = "hdfs";
}

if (!cmd.hasOption(Shell.USER)) {
throw new OzoneRestClientException(
"User name is needed in listVolume call.");
}

if (cmd.hasOption(Shell.USER)) {
userName = cmd.getOptionValue(Shell.USER);
} else {
Expand All @@ -80,7 +101,8 @@ protected void execute(CommandLine cmd)
client.setUserAuth(userName);
}

List<OzoneVolume> volumes = client.listVolumes(userName);
List<OzoneVolume> volumes = client.listVolumes(userName, prefix, maxKeys,
startVolume);
if (volumes != null) {
if (cmd.hasOption(Shell.VERBOSE)) {
System.out.printf("Found : %d volumes for user : %s %n", volumes.size(),
Expand Down
62 changes: 59 additions & 3 deletions ...hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View file
Expand Up @@ -20,11 +20,13 @@

import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
Expand All @@ -45,6 +47,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
Expand Down Expand Up @@ -80,7 +83,7 @@ public static void init() throws Exception {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);

cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();

Expand Down Expand Up @@ -123,8 +126,9 @@ public void testCreateDuplicateVolume() throws OzoneException {
client.createVolume("testvol", "bilbo", "100TB");
assertFalse(true);
} catch (OzoneException ex) {
// OZone will throw saying volume already exists
assertEquals(ex.getShortMessage(),"volumeAlreadyExists");
// Ozone will throw saying volume already exists
GenericTestUtils.assertExceptionContains(
Status.VOLUME_ALREADY_EXISTS.toString(), ex);
}
}

Expand Down Expand Up @@ -224,6 +228,58 @@ public void testListAllVolumes() throws OzoneException, IOException {
Assert.assertEquals(volCount / step , pagecount);
}

@Test
public void testListVolumes() throws OzoneException, IOException {
final int volCount = 20;
final String user1 = "test-user-a";
final String user2 = "test-user-b";

client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
// Create 20 volumes, 10 for user1 and another 10 for user2.
for (int x = 0; x < volCount; x++) {
String volumeName;
String userName;

if (x % 2 == 0) {
// create volume [test-vol0, test-vol2, ..., test-vol18] for user1
userName = user1;
volumeName = "test-vol" + x;
} else {
// create volume [test-vol1, test-vol3, ..., test-vol19] for user2
userName = user2;
volumeName = "test-vol" + x;
}
OzoneVolume vol = client.createVolume(volumeName, userName, "100TB");
assertNotNull(vol);
}

// list all the volumes belong to user1
List<OzoneVolume> volumeList = client.listVolumes(user1,
null, 100, StringUtils.EMPTY);
assertEquals(10, volumeList.size());
volumeList.stream()
.filter(item -> item.getOwnerName().equals(user1))
.collect(Collectors.toList());

// test max key parameter of listing volumes
volumeList = client.listVolumes(user1, null, 2, StringUtils.EMPTY);
assertEquals(2, volumeList.size());

// test prefix parameter of listing volumes
volumeList = client.listVolumes(user1, "test-vol10", 100,
StringUtils.EMPTY);
assertTrue(volumeList.size() == 1
&& volumeList.get(0).getVolumeName().equals("test-vol10"));

volumeList = client.listVolumes(user1, "test-vol1",
100, StringUtils.EMPTY);
assertEquals(5, volumeList.size());

// test start key parameter of listing volumes
volumeList = client.listVolumes(user2, null, 100, "test-vol17");
assertEquals(2, volumeList.size());
}

/**
* Returns a list of mocked {@link CloseableHttpClient} used for testing.
* The mocked client replaces the actual calls in
Expand Down

0 comments on commit ea5e24c

Please sign in to comment.