Skip to content

Commit

Permalink
HDDS-1670. Add limit support to /api/containers and /api/containers/{…
Browse files Browse the repository at this point in the history 
…id} endpoints (#954)
  • Loading branch information
@vivekratnavel @bharatviswa504
vivekratnavel authored and bharatviswa504 committed Jun 18, 2019
1 parent 335c1c9 commit fb1ce0d
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 29 deletions.
19 changes: 14 additions & 5 deletions ...zone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
View file
Expand Up @@ -21,17 +21,19 @@
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.stream.Collectors;

import javax.inject.Inject;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
Expand Down Expand Up @@ -72,10 +74,11 @@ public class ContainerKeyService {
* @return {@link Response}
*/
@GET
public Response getContainers() {
public Response getContainers(
@DefaultValue("-1") @QueryParam("limit") int limit) {
Map<Long, ContainerMetadata> containersMap;
try {
containersMap = containerDBServiceProvider.getContainers();
containersMap = containerDBServiceProvider.getContainers(limit);
} catch (IOException ioEx) {
throw new WebApplicationException(ioEx,
Response.Status.INTERNAL_SERVER_ERROR);
Expand All @@ -92,8 +95,10 @@ public Response getContainers() {
*/
@GET
@Path("/{id}")
public Response getKeysForContainer(@PathParam("id") Long containerId) {
Map<String, KeyMetadata> keyMetadataMap = new HashMap<>();
public Response getKeysForContainer(
@PathParam("id") Long containerId,
@DefaultValue("-1") @QueryParam("limit") int limit) {
Map<String, KeyMetadata> keyMetadataMap = new LinkedHashMap<>();
try {
Map<ContainerKeyPrefix, Integer> containerKeyPrefixMap =
containerDBServiceProvider.getKeyPrefixesForContainer(containerId);
Expand Down Expand Up @@ -143,6 +148,10 @@ public Response getKeysForContainer(@PathParam("id") Long containerId) {
Collections.singletonMap(containerKeyPrefix.getKeyVersion(),
blockIds));
} else {
// break the for loop if limit has been reached
if (keyMetadataMap.size() == limit) {
break;
}
KeyMetadata keyMetadata = new KeyMetadata();
keyMetadata.setBucket(omKeyInfo.getBucketName());
keyMetadata.setVolume(omKeyInfo.getVolumeName());
Expand Down
12 changes: 11 additions & 1 deletion ...one-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java
View file
Expand Up @@ -70,13 +70,23 @@ Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(long containerId)
throws IOException;

/**
* Get a Map of containerID, containerMetadata of all Containers.
* Get a Map of containerID, containerMetadata of all the Containers.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
Map<Long, ContainerMetadata> getContainers() throws IOException;

/**
* Get a Map of containerID, containerMetadata of Containers only for the
* given limit. If the limit is -1 or any integer <0, then return all
* the containers without any limit.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
Map<Long, ContainerMetadata> getContainers(int limit) throws IOException;

/**
* Delete an entry in the container DB.
* @param containerKeyPrefix container key prefix to be deleted.
Expand Down
27 changes: 24 additions & 3 deletions .../src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
View file
Expand Up @@ -22,7 +22,6 @@

import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;

Expand Down Expand Up @@ -139,7 +138,7 @@ public Integer getCountForForContainerKeyPrefix(
public Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(
long containerId) throws IOException {

Map<ContainerKeyPrefix, Integer> prefixes = new HashMap<>();
Map<ContainerKeyPrefix, Integer> prefixes = new LinkedHashMap<>();
TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
Integer>> containerIterator = containerKeyTable.iterator();
containerIterator.seek(new ContainerKeyPrefix(containerId));
Expand All @@ -166,13 +165,29 @@ public Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(
}

/**
* Iterate the DB to construct a Map of containerID -> containerMetadata.
* Get all the containers.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
@Override
public Map<Long, ContainerMetadata> getContainers() throws IOException {
// Set a negative limit to get all the containers.
return getContainers(-1);
}

/**
* Iterate the DB to construct a Map of containerID -> containerMetadata
* only for the given limit.
*
* Return all the containers if limit < 0.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
@Override
public Map<Long, ContainerMetadata> getContainers(int limit)
throws IOException {
Map<Long, ContainerMetadata> containers = new LinkedHashMap<>();
TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
Integer>> containerIterator = containerKeyTable.iterator();
Expand All @@ -181,6 +196,12 @@ public Map<Long, ContainerMetadata> getContainers() throws IOException {
Long containerID = keyValue.getKey().getContainerId();
Integer numberOfKeys = keyValue.getValue();

// break the loop if limit has been reached
// and one more new entity needs to be added to the containers map
if (containers.size() == limit && !containers.containsKey(containerID)) {
break;
}

// initialize containerMetadata with 0 as number of keys.
containers.computeIfAbsent(containerID, ContainerMetadata::new);
// increment number of keys for the containerID
Expand Down
51 changes: 31 additions & 20 deletions .../ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
View file
Expand Up @@ -20,6 +20,7 @@

import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.io.File;
Expand Down Expand Up @@ -200,56 +201,67 @@ protected void configure() {
@Test
public void testGetKeysForContainer() {

Response response = containerKeyService.getKeysForContainer(1L);
Response response = containerKeyService.getKeysForContainer(1L, -1);

Collection<KeyMetadata> keyMetadataList =
(Collection<KeyMetadata>) response.getEntity();
assertTrue(keyMetadataList.size() == 2);
assertEquals(keyMetadataList.size(), 2);

Iterator<KeyMetadata> iterator = keyMetadataList.iterator();

KeyMetadata keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_one"));
assertTrue(keyMetadata.getVersions().size() == 1);
assertTrue(keyMetadata.getBlockIds().size() == 1);
assertEquals(keyMetadata.getKey(), "key_one");
assertEquals(keyMetadata.getVersions().size(), 1);
assertEquals(keyMetadata.getBlockIds().size(), 1);
Map<Long, List<KeyMetadata.ContainerBlockMetadata>> blockIds =
keyMetadata.getBlockIds();
assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 101);
assertEquals(blockIds.get(0L).iterator().next().getLocalID(), 101);

keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_two"));
assertTrue(keyMetadata.getVersions().size() == 2);
assertEquals(keyMetadata.getKey(), "key_two");
assertEquals(keyMetadata.getVersions().size(), 2);
assertTrue(keyMetadata.getVersions().contains(0L) && keyMetadata
.getVersions().contains(1L));
assertTrue(keyMetadata.getBlockIds().size() == 2);
assertEquals(keyMetadata.getBlockIds().size(), 2);
blockIds = keyMetadata.getBlockIds();
assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 103);
assertTrue(blockIds.get(1L).iterator().next().getLocalID() == 104);
assertEquals(blockIds.get(0L).iterator().next().getLocalID(), 103);
assertEquals(blockIds.get(1L).iterator().next().getLocalID(), 104);

response = containerKeyService.getKeysForContainer(3L);
response = containerKeyService.getKeysForContainer(3L, -1);
keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
assertTrue(keyMetadataList.isEmpty());

// test if limit works as expected
response = containerKeyService.getKeysForContainer(1L, 1);
keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
assertEquals(keyMetadataList.size(), 1);
}

@Test
public void testGetContainers() {

Response response = containerKeyService.getContainers();
Response response = containerKeyService.getContainers(-1);

List<ContainerMetadata> containers = new ArrayList<>(
(Collection<ContainerMetadata>) response.getEntity());

assertTrue(containers.size() == 2);

Iterator<ContainerMetadata> iterator = containers.iterator();

ContainerMetadata containerMetadata = iterator.next();
assertTrue(containerMetadata.getContainerID() == 1L);
assertTrue(containerMetadata.getNumberOfKeys() == 3L);
assertEquals(containerMetadata.getContainerID(), 1L);
// Number of keys for CID:1 should be 3 because of two different versions
// of key_two stored in CID:1
assertEquals(containerMetadata.getNumberOfKeys(), 3L);

containerMetadata = iterator.next();
assertTrue(containerMetadata.getContainerID() == 2L);
assertTrue(containerMetadata.getNumberOfKeys() == 2L);
assertEquals(containerMetadata.getContainerID(), 2L);
assertEquals(containerMetadata.getNumberOfKeys(), 2L);

// test if limit works as expected
response = containerKeyService.getContainers(1);
containers = new ArrayList<>(
(Collection<ContainerMetadata>) response.getEntity());
assertEquals(containers.size(), 1);
}

/**
Expand All @@ -266,5 +278,4 @@ private OzoneConfiguration getTestOzoneConfiguration()
.getAbsolutePath());
return configuration;
}

}

0 comments on commit fb1ce0d

Please sign in to comment.