[#306] Add simple ContentSummary call

.travis.yml

@@ -17,6 +17,7 @@ matrix:
   - env: VERSION='-PhadoopVersion=2.7.0'
   - env: VERSION='-PhadoopVersion=2.8.0'
   - env: VERSION='-PhadoopVersion=2.9.0'
+  - env: VERSION='-PhadoopVersion=2.10.0'
   - env: VERSION='-PhadoopVersion=3.0.0'
   - env: VERSION='-PhadoopVersion=3.1.0'
 script:

build.gradle

@@ -157,7 +157,7 @@ dependencies {
     outboundDep group: 'com.sparkjava', name: 'spark-core', version: '2.5'
     outboundDep group: 'javax.servlet', name: 'javax.servlet-api', version: '3.1.0'
     outboundDep group: 'org.mortbay.jetty', name: 'jetty', version: '6.1.26'
-    outboundDep group: 'com.google.code.gson', name: 'gson', version: '2.2.4'
+    outboundDep group: 'com.google.code.gson', name: 'gson', version: '2.9.0'
     outboundDep group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.21'
     outboundDep group: 'log4j', name: 'log4j', version: '1.2.17'
     outboundDep group: 'javax.mail', name: 'mail', version: '1.4.7'

docs/REST_Endpoints/ContentSummary.md

@@ -0,0 +1,16 @@
+**ContentSummary:**
+
+`/contentSummary` is a GET only call that only READER users can access.
+It takes a required parameter `?path=<path>` to specify the location you wish to run the ContentSummary operation on.
+
+*Unlike other queries, `/contentSummary` will not take the NNA Query Lock and therefore will not make you wait for other queries to run. It will run in parallel, just like in an Active NameNode.*
+
+There are several optional parameters available to the histogram query:
+* `&useLock=<boolean>` if you wish to take the FSNamesystem lock as part of your query if you are seeing inconsistencies between histograms. This ensures the INode set will not change underneath mid-query.
+* `&useQueryLock=<boolean>` if you wish to take the NNA Query lock as part of your query. This will make you wait for subsequent queries to finish before yours runs.
+
+Response code is 200 and an XML representation of the HDFS configurations used to bootstrap.
+
+Response code of 403 means you are not authorized to view this endpoint.
+
+Response code of 404 means the path you specified does not exist.

docs/index.md

@@ -36,6 +36,7 @@ REST Endpoints
   * [AddDirectory](REST_Endpoints/AddDirectory.md)
   * [Bottom](REST_Endpoints/Bottom.md)
   * [Config](REST_Endpoints/Config.md)
+  * [ContentSummary](REST_Endpoints/ContentSummary.md)
   * [Credentials](REST_Endpoints/Credentials.md)
   * [Directories](REST_Endpoints/Directories.md)
   * [Divide](REST_Endpoints/Divide.md)

src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java

@@ -240,6 +240,7 @@ enum Endpoint {
     login,
     logout,
     endpoints,
+    contentSummary,
     credentials,
     loadingStatus,
     config,
@@ -325,6 +326,7 @@ enum Endpoint {
 
   EnumSet<Endpoint> READER_ENDPOINTS =
       EnumSet.of(
+          Endpoint.contentSummary,
           Endpoint.filter,
           Endpoint.histogram,
           Endpoint.histogram2,

src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLoader.java

@@ -545,20 +545,31 @@ public Collection<INode> getINodeSet(String set) {
   }
 
   /**
-   * Perform a content summary call against the underlying FSNamesystem.
+   * Perform a content summary call against the underlying FSNamesystem. Do not send Exception
+   * upwards - treat as empty.
    *
    * @param path the dir/file path to call content summary on
    * @return a summary of the subtree or file
    */
   public ContentSummary getContentSummary(String path) {
     try {
-      return namesystem.getContentSummary(path);
+      return getContentSummaryImpl(path);
     } catch (IOException e) {
       LOG.error("Error with getContentSummary.", e);
     }
     return new ContentSummary.Builder().fileCount(0L).directoryCount(0L).build();
   }
 
+  /**
+   * Perform a content summary call against the underlying FSNamesystem.
+   *
+   * @param path the dir/file path to call content summary on
+   * @return a summary of the subtree or file
+   */
+  public ContentSummary getContentSummaryImpl(String path) throws IOException {
+    return namesystem.getContentSummary(path);
+  }
+
   /**
    * Initializes the background thread that performs cached reporting for all users. Initializes the
    * background thread that refreshes Kerberos keytab for NNA process.

src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/WebServerMain.java

@@ -63,6 +63,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.server.namenode.Constants;
 import org.apache.hadoop.hdfs.server.namenode.Constants.AnalysisState;
@@ -1257,6 +1258,39 @@ public void init(
           }
         });
 
+    /* ContentSummary endpoint takes 1 set of "path" parameters and returns the full form output as an
+    `hdfs dfs -count` operation would. This is to provide folks with a simple alternative to doing count operations
+     on NNA. */
+    get(
+        "/contentSummary",
+        (req, res) -> {
+          res.header("Access-Control-Allow-Origin", "*");
+          res.header("Content-Type", "text/plain");
+          if (!nameNodeLoader.isInit()) {
+            return "";
+          }
+
+          String path = req.queryMap("path").value();
+          final Boolean useFsLock = req.queryMap("useLock").booleanValue();
+          final Boolean useQueryLock = req.queryMap("useQueryLock").booleanValue();
+
+          ContentSummary contentSummary;
+
+          if (useQueryLock != null && useQueryLock) {
+            lock.writeLock().lock();
+          }
+          nameNodeLoader.namesystemWriteLock(useFsLock);
+          try {
+            contentSummary = nameNodeLoader.getContentSummaryImpl(path);
+          } finally {
+            nameNodeLoader.namesystemWriteUnlock(useFsLock);
+            if (useQueryLock != null && useQueryLock) {
+              lock.writeLock().unlock();
+            }
+          }
+          return contentSummary.toString();
+        });
+
     /* Filter endpoint takes 1 set of "set", "filter", "sum" / "limit" parameters and returns either
     the list of file paths that pass the filters or the summation of the INode fields that pass the filters
     in PLAINTEXT form. */

src/main/java/org/apache/hadoop/hdfs/server/namenode/analytics/web/NamenodeAnalyticsMethods.java

@@ -81,6 +81,7 @@
 import javax.ws.rs.core.Response;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.server.namenode.Constants;
 import org.apache.hadoop.hdfs.server.namenode.Constants.Endpoint;
@@ -2001,6 +2002,62 @@ public Response histogram3() {
     }
   }
 
+  /**
+   * HISTOGRAM3 endpoint takes 1 set of "set", "filter", "type", and "sum" parameters and returns a
+   * histogram where the X-axis represents the "type" type and the Y-axis represents the "sum" type.
+   * Output types available dictated by "&histogramOutput=". Default is CHART form. This differs
+   * from Histogram endpoint in that it can output multiple sums and values in a single query.
+   */
+  @GET
+  @Path("/contentSummary")
+  @Produces({MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON})
+  public Response contentSummary() {
+    final NameNodeLoader nnLoader = (NameNodeLoader) context.getAttribute(NNA_NN_LOADER);
+    final ReentrantReadWriteLock queryLock =
+        (ReentrantReadWriteLock) context.getAttribute(NNA_QUERY_LOCK);
+    final AtomicBoolean cancelRequest = (AtomicBoolean) context.getAttribute(NNA_CANCEL_REQUEST);
+    try {
+      before();
+
+      if (!nnLoader.isInit()) {
+        PrintWriter writer = response.getWriter();
+        writer.write("");
+        writer.flush();
+        writer.close();
+        return Response.ok().build();
+      }
+
+      String path = request.getParameter("path");
+      final Boolean useFsLock = Boolean.parseBoolean(request.getParameter("useLock"));
+      final Boolean useQueryLock = Boolean.parseBoolean(request.getParameter("useQueryLock"));
+
+      ContentSummary contentSummary;
+
+      if (useQueryLock != null && useQueryLock) {
+        queryLock.writeLock().lock();
+      }
+      if (cancelRequest.get()) {
+        throw new IOException("Query cancelled.");
+      }
+      nnLoader.namesystemWriteLock(useFsLock);
+      try {
+        contentSummary = nnLoader.getContentSummaryImpl(path);
+      } finally {
+        nnLoader.namesystemWriteUnlock(useFsLock);
+        if (useQueryLock != null && useQueryLock) {
+          queryLock.writeLock().unlock();
+        }
+      }
+      return Response.ok(contentSummary.toString(), MediaType.TEXT_PLAIN).build();
+    } catch (RuntimeException rtex) {
+      return handleException(rtex);
+    } catch (Exception ex) {
+      return handleException(ex);
+    } finally {
+      after();
+    }
+  }
+
   /**
    * SUBMITOPERATION is a GET only call that only WRITER users can access. It take parameters like
    * /filter and schedules operations to perform.

src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java

@@ -48,9 +48,12 @@
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.FixMethodOrder;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.junit.runners.MethodSorters;
 
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public abstract class TestWithMiniClusterBase {
 
   protected static final SRandom RANDOM = new SRandom();
@@ -288,6 +291,43 @@ public void testRestartFetchNamespace() throws Exception {
     assertThat(restartedTxid, is(greaterThan(currentTxid + 99)));
   }
 
+  @Test
+  public void testUpdateSeenThenContentSummary() throws Exception {
+    HttpGet get =
+        new HttpGet(
+            "http://localhost:4567/contentSummary?path=/dir1&useLock=true&useQueryLock=true");
+    HttpResponse res = client.execute(hostPort, get);
+    List<String> output = IOUtils.readLines(res.getEntity().getContent());
+    System.out.println(output);
+    assertThat(res.getStatusLine().getStatusCode(), is(200));
+    assertThat(output.size(), is(1));
+    assertThat(output.get(0), containsString("none"));
+    assertThat(output.get(0), containsString("inf"));
+  }
+
+  @Test
+  public void testUpdateSeenThenContentSummaryFail() throws Exception {
+    HttpGet get =
+        new HttpGet(
+            "http://localhost:4567/contentSummary?path=/fakeDir&useLock=true&useQueryLock=true");
+    HttpResponse res = client.execute(hostPort, get);
+    List<String> output = IOUtils.readLines(res.getEntity().getContent());
+    System.out.println(output);
+    assertThat(res.getStatusLine().getStatusCode(), is(404));
+    assertThat(output.size(), is(1));
+  }
+
+  @Test
+  public void testUpdateSeenThenContentSummaryFailNoPath() throws Exception {
+    HttpGet get =
+        new HttpGet("http://localhost:4567/contentSummary?path&useLock=true&useQueryLock=true");
+    HttpResponse res = client.execute(hostPort, get);
+    List<String> output = IOUtils.readLines(res.getEntity().getContent());
+    System.out.println(output);
+    assertThat(res.getStatusLine().getStatusCode(), is(500));
+    assertThat(output.size(), is(greaterThan(1)));
+  }
+
   protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception {
     DistributedFileSystem fileSystem = (DistributedFileSystem) FileSystem.get(CONF);
     for (int i = 0; i < numOfFiles; i++) {