HDFS-7723. Quota By Storage Type namenode implemenation. (Contributed…

… by Xiaoyu Yao)

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -23,6 +23,9 @@ Trunk (Unreleased)
     HDFS-7720. Quota by Storage Type API, tools and ClientNameNode Protocol
     changes. (Xiaoyu Yao via Arpit Agarwal)
 
+    HDFS-7723. Quota By Storage Type namenode implemenation. (Xiaoyu Yao via
+    Arpit Agarwal)
+
   IMPROVEMENTS
 
     HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -421,6 +421,10 @@ public void setBlockPoolId(String blockPoolId) {
     }
   }
 
+  public BlockStoragePolicySuite getStoragePolicySuite() {
+    return storagePolicySuite;
+  }
+
   /** get the BlockTokenSecretManager */
   @VisibleForTesting
   public BlockTokenSecretManager getBlockTokenSecretManager() {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java

@@ -216,7 +216,9 @@ private synchronized void applyEdits(long firstTxId, int numTxns, byte[] data)
       }
       lastAppliedTxId = logLoader.getLastAppliedTxId();
 
-      FSImage.updateCountForQuota(getNamesystem().dir.getRoot()); // inefficient!
+      FSImage.updateCountForQuota(
+          getNamesystem().dir.getBlockStoragePolicySuite(),
+          getNamesystem().dir.rootDir); // inefficient!
     } finally {
       backupInputStream.clear();
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java

@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
 
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
@@ -116,4 +117,8 @@ public boolean yield() {
   public Content.Counts getCounts() {
     return counts;
   }
+
+  public BlockStoragePolicySuite getBlockStoragePolicySuite() {
+      return fsn.getBlockManager().getStoragePolicySuite();
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java

@@ -21,47 +21,97 @@
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
 import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.util.EnumCounters;
 
 /**
  * Quota feature for {@link INodeDirectory}. 
  */
 public final class DirectoryWithQuotaFeature implements INode.Feature {
   public static final long DEFAULT_NAMESPACE_QUOTA = Long.MAX_VALUE;
-  public static final long DEFAULT_DISKSPACE_QUOTA = HdfsConstants.QUOTA_RESET;
-
-  /** Name space quota */
-  private long nsQuota = DEFAULT_NAMESPACE_QUOTA;
-  /** Name space count */
-  private long namespace = 1L;
-  /** Disk space quota */
-  private long dsQuota = DEFAULT_DISKSPACE_QUOTA;
-  /** Disk space count */
-  private long diskspace = 0L;
-
-  DirectoryWithQuotaFeature(long nsQuota, long dsQuota) {
-    this.nsQuota = nsQuota;
-    this.dsQuota = dsQuota;
+  public static final long DEFAULT_SPACE_QUOTA = HdfsConstants.QUOTA_RESET;
+
+  private QuotaCounts quota;
+  private QuotaCounts usage;
+
+  public static class Builder {
+    private QuotaCounts quota;
+    private QuotaCounts usage;
+
+    public Builder() {
+      this.quota = new QuotaCounts.Builder().nameCount(DEFAULT_NAMESPACE_QUOTA).
+          spaceCount(DEFAULT_SPACE_QUOTA).typeCounts(DEFAULT_SPACE_QUOTA).build();
+      this.usage = new QuotaCounts.Builder().nameCount(1).build();
+    }
+
+    public Builder nameSpaceQuota(long nameSpaceQuota) {
+      this.quota.setNameSpace(nameSpaceQuota);
+      return this;
+    }
+
+    public Builder spaceQuota(long spaceQuota) {
+      this.quota.setDiskSpace(spaceQuota);
+      return this;
+    }
+
+    public Builder typeQuotas(EnumCounters<StorageType> typeQuotas) {
+      this.quota.setTypeSpaces(typeQuotas);
+      return this;
+    }
+
+    public Builder typeQuota(StorageType type, long quota) {
+      this.quota.setTypeSpace(type, quota);
+      return this;
+    }
+
+    public DirectoryWithQuotaFeature build() {
+      return new DirectoryWithQuotaFeature(this);
+    }
+  }
+
+  private DirectoryWithQuotaFeature(Builder builder) {
+    this.quota = builder.quota;
+    this.usage = builder.usage;
   }
 
   /** @return the quota set or -1 if it is not set. */
-  Quota.Counts getQuota() {
-    return Quota.Counts.newInstance(nsQuota, dsQuota);
+  QuotaCounts getQuota() {
+    return new QuotaCounts.Builder().quotaCount(this.quota).build();
   }
-  
+
   /** Set this directory's quota
    * 
    * @param nsQuota Namespace quota to be set
    * @param dsQuota Diskspace quota to be set
+   * @param type Storage type quota to be set
+   * * To set traditional space/namespace quota, type must be null
    */
+  void setQuota(long nsQuota, long dsQuota, StorageType type) {
+    if (type != null) {
+      this.quota.setTypeSpace(type, dsQuota);
+    } else {
+      setQuota(nsQuota, dsQuota);
+    }
+  }
+
   void setQuota(long nsQuota, long dsQuota) {
-    this.nsQuota = nsQuota;
-    this.dsQuota = dsQuota;
+    this.quota.setNameSpace(nsQuota);
+    this.quota.setDiskSpace(dsQuota);
   }
-
-  Quota.Counts addNamespaceDiskspace(Quota.Counts counts) {
-    counts.add(Quota.NAMESPACE, namespace);
-    counts.add(Quota.DISKSPACE, diskspace);
+
+  void setQuota(long dsQuota, StorageType type) {
+    this.quota.setTypeSpace(type, dsQuota);
+  }
+
+  // Set in a batch only during FSImage load
+  void setQuota(EnumCounters<StorageType> typeQuotas) {
+    this.quota.setTypeSpaces(typeQuotas);
+  }
+
+  QuotaCounts addNamespaceDiskspace(QuotaCounts counts) {
+    counts.add(this.usage);
     return counts;
   }
 
@@ -76,98 +126,146 @@ ContentSummaryComputationContext computeContentSummary(final INodeDirectory dir,
     }
     return summary;
   }
-  
+
   private void checkDiskspace(final INodeDirectory dir, final long computed) {
-    if (-1 != getQuota().get(Quota.DISKSPACE) && diskspace != computed) {
+    if (-1 != quota.getDiskSpace() && usage.getDiskSpace() != computed) {
       NameNode.LOG.error("BUG: Inconsistent diskspace for directory "
-          + dir.getFullPathName() + ". Cached = " + diskspace
+          + dir.getFullPathName() + ". Cached = " + usage.getDiskSpace()
           + " != Computed = " + computed);
     }
   }
 
-  void addSpaceConsumed(final INodeDirectory dir, final long nsDelta,
-      final long dsDelta, boolean verify) throws QuotaExceededException {
-    if (dir.isQuotaSet()) { 
-      // The following steps are important: 
+  void addSpaceConsumed(final INodeDirectory dir, final QuotaCounts counts,
+      boolean verify) throws QuotaExceededException {
+    if (dir.isQuotaSet()) {
+      // The following steps are important:
       // check quotas in this inode and all ancestors before changing counts
       // so that no change is made if there is any quota violation.
-
       // (1) verify quota in this inode
       if (verify) {
-        verifyQuota(nsDelta, dsDelta);
+        verifyQuota(counts);
       }
-      // (2) verify quota and then add count in ancestors 
-      dir.addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
+      // (2) verify quota and then add count in ancestors
+      dir.addSpaceConsumed2Parent(counts, verify);
       // (3) add count in this inode
-      addSpaceConsumed2Cache(nsDelta, dsDelta);
+      addSpaceConsumed2Cache(counts);
     } else {
-      dir.addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
+      dir.addSpaceConsumed2Parent(counts, verify);
     }
   }
 
-  /** Update the size of the tree
+  /** Update the space/namespace/type usage of the tree
    * 
-   * @param nsDelta the change of the tree size
-   * @param dsDelta change to disk space occupied
+   * @param delta the change of the namespace/space/type usage
    */
-  public void addSpaceConsumed2Cache(long nsDelta, long dsDelta) {
-    namespace += nsDelta;
-    diskspace += dsDelta;
+  public void addSpaceConsumed2Cache(QuotaCounts delta) {
+    usage.add(delta);
   }
-  
+
   /** 
    * Sets namespace and diskspace take by the directory rooted 
    * at this INode. This should be used carefully. It does not check 
    * for quota violations.
    * 
    * @param namespace size of the directory to be set
    * @param diskspace disk space take by all the nodes under this directory
+   * @param typeUsed counters of storage type usage
    */
-  void setSpaceConsumed(long namespace, long diskspace) {
-    this.namespace = namespace;
-    this.diskspace = diskspace;
+  void setSpaceConsumed(long namespace, long diskspace,
+      EnumCounters<StorageType> typeUsed) {
+    usage.setNameSpace(namespace);
+    usage.setDiskSpace(diskspace);
+    usage.setTypeSpaces(typeUsed);
   }
-
+
+  void setSpaceConsumed(QuotaCounts c) {
+    usage.setNameSpace(c.getNameSpace());
+    usage.setDiskSpace(c.getDiskSpace());
+    usage.setTypeSpaces(c.getTypeSpaces());
+  }
+
   /** @return the namespace and diskspace consumed. */
-  public Quota.Counts getSpaceConsumed() {
-    return Quota.Counts.newInstance(namespace, diskspace);
+  public QuotaCounts getSpaceConsumed() {
+    return new QuotaCounts.Builder().quotaCount(usage).build();
   }
 
   /** Verify if the namespace quota is violated after applying delta. */
   private void verifyNamespaceQuota(long delta) throws NSQuotaExceededException {
-    if (Quota.isViolated(nsQuota, namespace, delta)) {
-      throw new NSQuotaExceededException(nsQuota, namespace + delta);
+    if (Quota.isViolated(quota.getNameSpace(), usage.getNameSpace(), delta)) {
+      throw new NSQuotaExceededException(quota.getNameSpace(),
+          usage.getNameSpace() + delta);
     }
   }
   /** Verify if the diskspace quota is violated after applying delta. */
   private void verifyDiskspaceQuota(long delta) throws DSQuotaExceededException {
-    if (Quota.isViolated(dsQuota, diskspace, delta)) {
-      throw new DSQuotaExceededException(dsQuota, diskspace + delta);
+    if (Quota.isViolated(quota.getDiskSpace(), usage.getDiskSpace(), delta)) {
+      throw new DSQuotaExceededException(quota.getDiskSpace(),
+          usage.getDiskSpace() + delta);
+    }
+  }
+
+  private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
+      throws QuotaByStorageTypeExceededException {
+    if (!isQuotaByStorageTypeSet()) {
+      return;
+    }
+    for (StorageType t: StorageType.getTypesSupportingQuota()) {
+      if (!isQuotaByStorageTypeSet(t)) {
+        continue;
+      }
+      if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
+          typeDelta.get(t))) {
+        throw new QuotaByStorageTypeExceededException(
+          quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
+      }
     }
   }
 
   /**
-   * @throws QuotaExceededException if namespace or diskspace quotas is
-   *         violated after applying the deltas.
+   * @throws QuotaExceededException if namespace, diskspace or storage type quotas
+   * is violated after applying the deltas.
    */
-  void verifyQuota(long nsDelta, long dsDelta) throws QuotaExceededException {
-    verifyNamespaceQuota(nsDelta);
-    verifyDiskspaceQuota(dsDelta);
+  void verifyQuota(QuotaCounts counts) throws QuotaExceededException {
+    verifyNamespaceQuota(counts.getNameSpace());
+    verifyDiskspaceQuota(counts.getDiskSpace());
+    verifyQuotaByStorageType(counts.getTypeSpaces());
   }
-  
+
   boolean isQuotaSet() {
-    return nsQuota >= 0 || dsQuota >= 0;
+    return quota.anyNsSpCountGreaterOrEqual(0) ||
+        quota.anyTypeCountGreaterOrEqual(0);
+  }
+
+  boolean isQuotaByStorageTypeSet() {
+    return quota.anyTypeCountGreaterOrEqual(0);
+  }
+
+  boolean isQuotaByStorageTypeSet(StorageType t) {
+    return quota.getTypeSpace(t) >= 0;
   }
 
   private String namespaceString() {
-    return "namespace: " + (nsQuota < 0? "-": namespace + "/" + nsQuota);
+    return "namespace: " + (quota.getNameSpace() < 0? "-":
+        usage.getNameSpace() + "/" + quota.getNameSpace());
   }
   private String diskspaceString() {
-    return "diskspace: " + (dsQuota < 0? "-": diskspace + "/" + dsQuota);
+    return "diskspace: " + (quota.getDiskSpace() < 0? "-":
+        usage.getDiskSpace() + "/" + quota.getDiskSpace());
   }
-
+
+  private String quotaByStorageTypeString() {
+    StringBuilder sb = new StringBuilder();
+    for (StorageType t : StorageType.getTypesSupportingQuota()) {
+      sb.append("StorageType: " + t +
+          (quota.getTypeSpace(t) < 0? "-":
+          usage.getTypeSpace(t) + "/" + usage.getTypeSpace(t)));
+    }
+    return sb.toString();
+  }
+
   @Override
   public String toString() {
-    return "Quota[" + namespaceString() + ", " + diskspaceString() + "]";
+    return "Quota[" + namespaceString() + ", " + diskspaceString() +
+        ", " + quotaByStorageTypeString() + "]";
   }
-}
+}