os/chain_xattr: stripe shortish xattrs over small chunks for XFS

XFS has a hard limit of 255 (or 254?) bytes for xattrs to be inlined in the
inode due to a single byte for the length in the on-disk format.  If we
have an xattr that is a bit bigger than that it will get kicked out into a
separate extent, but if we stripe it across a few shorter xattrs it can
be inlined when the xfs inode is e.g. 2K.

Adjust the chain_xattr logic to capture this case.  Note that we are doing
this unconditionally regardless of fs type, but that is probably okay
given that most users use XFS and the cost isn't huge.

Reported-by: Ning Yao <yaoning@ruijie.com.cn>
Signed-off-by: Sage Weil <sage@redhat.com>

src/os/chain_xattr.cc

@@ -116,7 +116,8 @@ static int getxattr_len(const char *fn, const char *name)
       break;
     total += r;
     i++;
-  } while (r == CHAIN_XATTR_MAX_BLOCK_LEN);
+  } while (r == CHAIN_XATTR_MAX_BLOCK_LEN ||
+	   r == CHAIN_XATTR_SHORT_BLOCK_LEN);
 
   return total;
 }
@@ -150,7 +151,8 @@ int chain_getxattr(const char *fn, const char *name, void *val, size_t size)
       pos += r;
 
     i++;
-  } while (size && r == CHAIN_XATTR_MAX_BLOCK_LEN);
+  } while (size && (r == CHAIN_XATTR_MAX_BLOCK_LEN ||
+		    r == CHAIN_XATTR_SHORT_BLOCK_LEN));
 
   if (r >= 0) {
     ret = pos;
@@ -182,7 +184,8 @@ static int chain_fgetxattr_len(int fd, const char *name)
       break;
     total += r;
     i++;
-  } while (r == CHAIN_XATTR_MAX_BLOCK_LEN);
+  } while (r == CHAIN_XATTR_MAX_BLOCK_LEN ||
+	   r == CHAIN_XATTR_SHORT_BLOCK_LEN);
 
   return total;
 }
@@ -216,7 +219,8 @@ int chain_fgetxattr(int fd, const char *name, void *val, size_t size)
       pos += r;
 
     i++;
-  } while (size && r == CHAIN_XATTR_MAX_BLOCK_LEN);
+  } while (size && (r == CHAIN_XATTR_MAX_BLOCK_LEN ||
+		    r == CHAIN_XATTR_SHORT_BLOCK_LEN));
 
   if (r >= 0) {
     ret = pos;
@@ -236,14 +240,24 @@ int chain_fgetxattr(int fd, const char *name, void *val, size_t size)
 
 // setxattr
 
+static int get_xattr_block_len(size_t size)
+{
+  if (size < CHAIN_XATTR_SHORT_LEN_THRESHOLD)
+    // this may fit in the inode; stripe over short attrs so that XFS
+    // won't kick it out.
+    return CHAIN_XATTR_SHORT_BLOCK_LEN;
+  return CHAIN_XATTR_MAX_BLOCK_LEN;
+}
+
 int chain_setxattr(const char *fn, const char *name, const void *val, size_t size)
 {
   int i = 0, pos = 0;
   char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
   int ret = 0;
+  size_t max_chunk_size = get_xattr_block_size(size);
 
   do {
-    size_t chunk_size = (size < CHAIN_XATTR_MAX_BLOCK_LEN ? size : CHAIN_XATTR_MAX_BLOCK_LEN);
+    size_t chunk_size = (size < max_chunk_size ? size : max_chunk_size);
     get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
     size -= chunk_size;
 
@@ -276,9 +290,10 @@ int chain_fsetxattr(int fd, const char *name, const void *val, size_t size)
   int i = 0, pos = 0;
   char raw_name[CHAIN_XATTR_MAX_NAME_LEN * 2 + 16];
   int ret = 0;
+  size_t max_chunk_size = get_xattr_block_size(size);
 
   do {
-    size_t chunk_size = (size < CHAIN_XATTR_MAX_BLOCK_LEN ? size : CHAIN_XATTR_MAX_BLOCK_LEN);
+    size_t chunk_size = (size < max_chunk_size ? size : max_chunk_size);
     get_raw_xattr_name(name, i, raw_name, sizeof(raw_name));
     size -= chunk_size;
 

src/os/chain_xattr.h

@@ -11,6 +11,12 @@
 #define CHAIN_XATTR_MAX_NAME_LEN  128
 #define CHAIN_XATTR_MAX_BLOCK_LEN 2048
 
+/*
+ * XFS will only inline xattrs < 255 bytes, so for xattrs that are
+ * likely to fit in the inode, stripe over short xattrs.
+ */
+#define CHAIN_XATTR_SHORT_BLOCK_LEN 250
+#define CHAIN_XATTR_SHORT_LEN_THRESHOLD 1800
 
 // wrappers to hide annoying errno handling.