HDFS-17084. Utilize StringTable for numerable XAttributes

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -3241,8 +3241,32 @@ public AclStatus getAclStatus(Path path) throws IOException {
    */
   public void setXAttr(Path path, String name, byte[] value)
       throws IOException {
-    setXAttr(path, name, value, EnumSet.of(XAttrSetFlag.CREATE,
-        XAttrSetFlag.REPLACE));
+    setXAttr(path, name, value, false);
+  }
+
+  /**
+   * Set an xattr of a file or directory.
+   * The name must be prefixed with the namespace followed by ".". For example,
+   * "user.attr".
+   * <p>
+   * Refer to the HDFS extended attributes user documentation for details.
+   *
+   * @param path Path to modify
+   * @param name xattr name.
+   * @param value xattr value.
+   * @param enumValue if value is enumerable
+   * @throws IOException IO failure
+   * @throws UnsupportedOperationException if the operation is unsupported
+   *         (default outcome).
+   */
+  public void setXAttr(Path path, String name, byte[] value, boolean enumValue)
+          throws IOException {
+    EnumSet<XAttrSetFlag> flags = EnumSet.of(XAttrSetFlag.CREATE,
+            XAttrSetFlag.REPLACE);
+    if (enumValue) {
+      flags.add(XAttrSetFlag.ENUM_VALUE);
+    }
+    setXAttr(path, name, value, flags);
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -626,6 +626,12 @@ public void setXAttr(Path path, String name, byte[] value)
     fs.setXAttr(path, name, value);
   }
 
+  @Override
+  public void setXAttr(Path path, String name, byte[] value, boolean numerable)
+      throws IOException {
+    fs.setXAttr(path, name, value, numerable);
+  }
+
   @Override
   public void setXAttr(Path path, String name, byte[] value,
       EnumSet<XAttrSetFlag> flag) throws IOException {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/XAttrSetFlag.java

@@ -38,7 +38,13 @@ public enum XAttrSetFlag {
    * Replace a existing xattr.
    * If the xattr does not exist, exception will be thrown.
    */
-  REPLACE((short) 0x02);
+  REPLACE((short) 0x02),
+
+  /**
+   * Value is enumerable.
+   * Value will be stored efficiently.
+   */
+  ENUM_VALUE((short) 0x04);
 
   private final short flag;
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java

@@ -133,30 +133,38 @@ private void printXAttr(String name, byte[] value) throws IOException{
    */
   public static class SetfattrCommand extends FsCommand {
     public static final String NAME = SET_FATTR;
-    public static final String USAGE = "{-n name [-v value] | -x name} <path>";
+    public static final String USAGE = "{-n name [-v value [-e]] | -x name} <path>";
     public static final String DESCRIPTION =
-      "Sets an extended attribute name and value for a file or directory.\n" +
-      "-n name: The extended attribute name.\n" +
-      "-v value: The extended attribute value. There are three different " +
-      "encoding methods for the value. If the argument is enclosed in double " +
-      "quotes, then the value is the string inside the quotes. If the " +
-      "argument is prefixed with 0x or 0X, then it is taken as a hexadecimal " +
-      "number. If the argument begins with 0s or 0S, then it is taken as a " +
-      "base64 encoding.\n" +
-      "-x name: Remove the extended attribute.\n" +
-      "<path>: The file or directory.\n";
+        "Sets an extended attribute name and value for a file or directory.\n" +
+        "-n name: The extended attribute name.\n" +
+        "-v value: The extended attribute value. There are three different " +
+        "encoding methods for the value. If the argument is enclosed in double " +
+        "quotes, then the value is the string inside the quotes. If the " +
+        "argument is prefixed with 0x or 0X, then it is taken as a hexadecimal " +
+        "number. If the argument begins with 0s or 0S, then it is taken as a " +
+        "base64 encoding.\n" +
+        "-e: if set, means the value is enumerable, HDFS will try to store it " +
+        "efficiently.\n" +
+        "-x name: Remove the extended attribute.\n" +
+        "<path>: The file or directory.\n";
 
     private String name = null;
     private byte[] value = null;
     private String xname = null;
+    private boolean enumValue = false;
 
     @Override
     protected void processOptions(LinkedList<String> args) throws IOException {
       name = StringUtils.popOptionWithArgument("-n", args);
       String v = StringUtils.popOptionWithArgument("-v", args);
+      enumValue = StringUtils.popOption("-e", args);
       if (v != null) {
         value = XAttrCodec.decodeValue(v);
       }
+      if (enumValue && v == null) {
+        throw new HadoopIllegalArgumentException(
+                "Can not specify '-e' when '-v' is empty.");
+      }
       xname = StringUtils.popOptionWithArgument("-x", args);
 
       if (name != null && xname != null) {
@@ -179,7 +187,7 @@ protected void processOptions(LinkedList<String> args) throws IOException {
     @Override
     protected void processPath(PathData item) throws IOException {
       if (name != null) {
-        item.fs.setXAttr(item.path, name, value);
+        item.fs.setXAttr(item.path, name, value, enumValue);
       } else if (xname != null) {
         item.fs.removeXAttr(item.path, xname);
       }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java

@@ -198,6 +198,9 @@ public void removeAclEntries(Path path, List<AclEntry> aclSpec)
     public void setXAttr(Path path, String name, byte[] value)
         throws IOException;
 
+    void setXAttr(Path path, String name, byte[] value, boolean numerable)
+        throws IOException;
+
     public void setXAttr(Path path, String name, byte[] value,
         EnumSet<XAttrSetFlag> flag) throws IOException;
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/XAttr.java

@@ -68,11 +68,13 @@ public enum NameSpace {
   private final NameSpace ns;
   private final String name;
   private final byte[] value;
+  private final boolean enumerable;
 
   public static class Builder {
     private NameSpace ns = NameSpace.USER;
     private String name;
     private byte[] value;
+    private boolean numerable;
 
     public Builder setNameSpace(NameSpace ns) {
       this.ns = ns;
@@ -89,15 +91,21 @@ public Builder setValue(byte[] value) {
       return this;
     }
 
+    public Builder setEnumerable(boolean isNumerable) {
+      this.numerable = isNumerable;
+      return this;
+    }
+
     public XAttr build() {
-      return new XAttr(ns, name, value);
+      return new XAttr(ns, name, value, numerable);
     }
   }
 
-  private XAttr(NameSpace ns, String name, byte[] value) {
+  private XAttr(NameSpace ns, String name, byte[] value, boolean enumerable) {
     this.ns = ns;
     this.name = name;
     this.value = value;
+    this.enumerable = enumerable;
   }
 
   public NameSpace getNameSpace() {
@@ -112,12 +120,17 @@ public byte[] getValue() {
     return value;
   }
 
+  public boolean isEnumerable() {
+    return enumerable;
+  }
+
   @Override
   public int hashCode() {
     return new HashCodeBuilder(811, 67)
         .append(name)
         .append(ns)
         .append(value)
+        .append(enumerable)
         .toHashCode();
   }
 
@@ -133,6 +146,7 @@ public boolean equals(Object obj) {
         .append(ns, rhs.ns)
         .append(name, rhs.name)
         .append(value, rhs.value)
+        .append(enumerable, rhs.enumerable)
         .isEquals();
   }
 
@@ -152,12 +166,13 @@ public boolean equalsIgnoreValue(Object obj) {
     return new EqualsBuilder()
         .append(ns, rhs.ns)
         .append(name, rhs.name)
+        .append(isEnumerable(), rhs.isEnumerable())
         .isEquals();
   }
 
   @Override
   public String toString() {
-    return "XAttr [ns=" + ns + ", name=" + name + ", value="
+    return "XAttr [ns=" + ns + ", name=" + name + ", enumerable-" + enumerable + ", value="
         + Arrays.toString(value) + "]";
   }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java

@@ -2906,7 +2906,7 @@ public void setXAttr(String src, String name, byte[] value,
       EnumSet<XAttrSetFlag> flag) throws IOException {
     checkOpen();
     try (TraceScope ignored = newPathTraceScope("setXAttr", src)) {
-      namenode.setXAttr(src, XAttrHelper.buildXAttr(name, value), flag);
+      namenode.setXAttr(src, XAttrHelper.buildXAttr(name, value, flag), flag);
     } catch (RemoteException re) {
       throw re.unwrapRemoteException(AccessControlException.class,
           FileNotFoundException.class,

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java

@@ -17,13 +17,15 @@
  */
 package org.apache.hadoop.hdfs;
 
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.XAttr.NameSpace;
+import org.apache.hadoop.fs.XAttrSetFlag;
 import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.StringUtils;
 
@@ -47,6 +49,10 @@ public static XAttr buildXAttr(String name) {
    * Both name and namespace are case sensitive.
    */
   public static XAttr buildXAttr(String name, byte[] value) {
+    return buildXAttr(name, value, null);
+  }
+
+  public static XAttr buildXAttr(String name, byte[] value, final EnumSet<XAttrSetFlag> flag) {
     Preconditions.checkNotNull(name, "XAttr name cannot be null.");
 
     final int prefixIndex = name.indexOf(".");
@@ -78,8 +84,13 @@ public static XAttr buildXAttr(String name, byte[] value) {
           "prefixed with user/trusted/security/system/raw, followed by a '.'");
     }
 
+    boolean isNumerable = false;
+    if (flag != null && flag.contains(XAttrSetFlag.ENUM_VALUE)) {
+      isNumerable = true;
+    }
+
     return (new XAttr.Builder()).setNameSpace(ns).setName(name.
-        substring(prefixIndex + 1)).setValue(value).build();
+        substring(prefixIndex + 1)).setValue(value).setEnumerable(isNumerable).build();
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java

@@ -1106,6 +1106,9 @@ public static XAttrProto convertXAttrProto(XAttr a) {
     if (a.getValue() != null) {
       builder.setValue(getByteString(a.getValue()));
     }
+    if (a.isEnumerable()) {
+      builder.setEnumerable(true);
+    }
     return builder.build();
   }
 
@@ -1353,6 +1356,9 @@ public static int convert(EnumSet<XAttrSetFlag> flag) {
     if (flag.contains(XAttrSetFlag.REPLACE)) {
       value |= XAttrSetFlagProto.XATTR_REPLACE.getNumber();
     }
+    if (flag.contains(XAttrSetFlag.ENUM_VALUE)) {
+      value |= XAttrSetFlagProto.XATTR_ENUMERABLE.getNumber();
+    }
     return value;
   }
 
@@ -2929,6 +2935,10 @@ public static EnumSet<XAttrSetFlag> convert(int flag) {
         XAttrSetFlagProto.XATTR_REPLACE_VALUE) {
       result.add(XAttrSetFlag.REPLACE);
     }
+    if ((flag & XAttrSetFlagProto.XATTR_ENUMERABLE.getNumber()) ==
+        XAttrSetFlagProto.XATTR_ENUMERABLE.getNumber()) {
+      result.add(XAttrSetFlag.ENUM_VALUE);
+    }
     return result;
   }
 
@@ -2941,6 +2951,9 @@ public static XAttr convertXAttr(XAttrProto a) {
     if (a.hasValue()) {
       builder.setValue(a.getValue().toByteArray());
     }
+    if (a.hasEnumerable()) {
+      builder.setEnumerable(a.getEnumerable());
+    }
     return builder.build();
   }
 

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/xattr.proto

@@ -33,11 +33,13 @@ message XAttrProto {
   required XAttrNamespaceProto namespace = 1;
   required string name = 2;
   optional bytes value = 3;
+  optional bool enumerable = 4;
 }
 
 enum XAttrSetFlagProto {
   XATTR_CREATE     = 0x01;
   XATTR_REPLACE    = 0x02;
+  XATTR_ENUMERABLE = 0x04;
 }
 
 message SetXAttrRequestProto {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java

@@ -5587,6 +5587,10 @@ private static void appendXAttrsToXml(ContentHandler contentHandler,
       XMLUtils.addSaxString(contentHandler, "NAMESPACE",
           xAttr.getNameSpace().toString());
       XMLUtils.addSaxString(contentHandler, "NAME", xAttr.getName());
+      if (xAttr.isEnumerable()) {
+        XMLUtils.addSaxString(contentHandler, "NUMERABLE",
+            String.valueOf(xAttr.isEnumerable()));
+      }
       if (xAttr.getValue() != null) {
         try {
           XMLUtils.addSaxString(contentHandler, "VALUE",
@@ -5610,7 +5614,8 @@ private static List<XAttr> readXAttrsFromXml(Stanza st)
     for (Stanza a: stanzas) {
       XAttr.Builder builder = new XAttr.Builder();
       builder.setNameSpace(XAttr.NameSpace.valueOf(a.getValue("NAMESPACE"))).
-          setName(a.getValue("NAME"));
+          setName(a.getValue("NAME")).
+          setEnumerable(Boolean.parseBoolean(a.getValueOrNull("NUMERABLE")));
       String v = a.getValueOrNull("VALUE");
       if (v != null) {
         try {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java

@@ -92,6 +92,9 @@ public final class FSImageFormatPBINode {
   /* See the comments in fsimage.proto for an explanation of the following. */
   public static final int XATTR_NAMESPACE_EXT_OFFSET = 5;
   public static final int XATTR_NAMESPACE_EXT_MASK = 1;
+  public static final int XATTR_NUMERABLE_OFFSET = 4;
+  public static final int XATTR_NUMERABLE_MASK = 1;
+  public static final int XATTR_VALUE_MASK = (1 << 24) - 1;
 
   private static final Logger LOG =
       LoggerFactory.getLogger(FSImageFormatPBINode.class);
@@ -121,10 +124,16 @@ public static List<XAttr> loadXAttrs(
       for (XAttrCompactProto xAttrCompactProto : proto.getXAttrsList()) {
         int v = xAttrCompactProto.getName();
         byte[] value = null;
-        if (xAttrCompactProto.getValue() != null) {
-          value = xAttrCompactProto.getValue().toByteArray();
+        if (XAttrFormat.isNumerable(v)) {
+          assert xAttrCompactProto.hasValueInt();
+          int valueInt = xAttrCompactProto.getValueInt();
+          b.add(XAttrFormat.toXAttr(v, valueInt, stringTable));
+        } else {
+          if (xAttrCompactProto.getValue() != null) {
+            value = xAttrCompactProto.getValue().toByteArray();
+          }
+          b.add(XAttrFormat.toXAttr(v, value, stringTable));
         }
-        b.add(XAttrFormat.toXAttr(v, value, stringTable));
       }
 
       return b;
@@ -648,8 +657,12 @@ private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f) {
             newBuilder();
         int v = XAttrFormat.toInt(a);
         xAttrCompactBuilder.setName(v);
-        if (a.getValue() != null) {
-          xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue()));
+        if (a.isEnumerable()) {
+          xAttrCompactBuilder.setValueInt(XAttrValueFormat.toInt(a.getValue()));
+        } else {
+          if (a.getValue() != null) {
+            xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue()));
+          }
         }
         b.addXAttrs(xAttrCompactBuilder.build());
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java

@@ -30,7 +30,7 @@ public enum SerialNumberManager {
   GLOBAL(),
   USER(PermissionStatusFormat.USER, AclEntryStatusFormat.NAME),
   GROUP(PermissionStatusFormat.GROUP, AclEntryStatusFormat.NAME),
-  XATTR(XAttrFormat.NAME);
+  XATTR(XAttrFormat.NAME, XAttrValueFormat.VALUE);
 
   private static final SerialNumberManager[] values = values();
   private static final int maxEntryBits;