FSAL_CEPH: track export id as part of the host handle

Typically, ganesha mixes the export_id in with the filehandle before
presenting it to clients, but then strips that out before converting it
to a host handle.

When a handle is reachable via two or more different exports, most FSALs
just keep a single cache entry for it. For FSAL_CEPH, we need to keep
them separate, as the different exports may use different cephx creds.

Add an export_id field to the host handle, and fill out the field from
the op_ctx when converting it from the wire handle. This leaves the
wire handle unchanged, but allows us to keep different cache entries
when the same inode shows up in different exports.

Change-Id: I6426a367061b880ee61c19586f4f54a3bc55ef0a
Signed-off-by: Jeff Layton <jlayton@redhat.com>

src/FSAL/FSAL_CEPH/export.c

@@ -187,7 +187,17 @@ static fsal_status_t wire_to_host(struct fsal_export *exp_hdl,
 		/* Digested Handles */
 	case FSAL_DIGEST_NFSV3:
 	case FSAL_DIGEST_NFSV4:
-		/* wire handles */
+		/*
+		 * Ganesha automatically mixes the export_id in with the
+		 * filehandle and strips that out before calling this
+		 * function.
+		 *
+		 * Most FSALs don't factor in the export_id with the handle_key,
+		 * but we want to do that for FSAL_CEPH, primarily because we
+		 * want to do accesses via different exports via different cephx
+		 * creds. Mix the export_id back in here.
+		 */
+		key->export_id = op_ctx->ctx_export->export_id;
 		fh_desc->len = sizeof(*key);
 		break;
 	default:
@@ -220,7 +230,7 @@ static fsal_status_t create_handle(struct fsal_export *export_pub,
 	/* FSAL status to return */
 	fsal_status_t status = { ERR_FSAL_NO_ERROR, 0 };
 	/* The FSAL specific portion of the handle received by the client */
-	struct ceph_handle_key *key = desc->addr;
+	struct ceph_host_handle *key = desc->addr;
 	/* Ceph return code */
 	int rc = 0;
 	/* Stat buffer */

src/FSAL/FSAL_CEPH/handle.c

@@ -2155,8 +2155,8 @@ static void ceph_deleg_cb(Fh *fh, void *vhdl)
 	struct ceph_handle *hdl =
 		container_of(obj_hdl, struct ceph_handle, handle);
 	struct gsh_buffdesc key = {
-		.addr = &hdl->key,
-		.len = sizeof(hdl->key)
+		.addr = &hdl->key.hhdl,
+		.len = sizeof(hdl->key.hhdl)
 	};
 
 	LogDebug(COMPONENT_FSAL, "Recalling delegations on %p", hdl);
@@ -2520,14 +2520,15 @@ ceph_fsal_handle_to_wire(const struct fsal_obj_handle *handle_pub,
 		/* Digested Handles */
 	case FSAL_DIGEST_NFSV3:
 	case FSAL_DIGEST_NFSV4:
-		if (fh_desc->len < sizeof(handle->key)) {
+		if (fh_desc->len < sizeof(handle->key.hhdl)) {
 			LogMajor(COMPONENT_FSAL,
 				 "digest_handle: space too small for handle.  Need %zu, have %zu",
-				 sizeof(handle->key), fh_desc->len);
+				 sizeof(handle->key.hhdl), fh_desc->len);
 			return fsalstat(ERR_FSAL_TOOSMALL, 0);
 		} else {
-			fh_desc->len = sizeof(handle->key);
-			memcpy(fh_desc->addr, &handle->key, fh_desc->len);
+			fh_desc->len = sizeof(handle->key.hhdl);
+			memcpy(fh_desc->addr, &handle->key.hhdl,
+			       fh_desc->len);
 		}
 		break;
 

src/FSAL/FSAL_CEPH/internal.c

@@ -77,11 +77,12 @@ void construct_handle(const struct ceph_statx *stx, struct Inode *i,
 
 	constructing = gsh_calloc(1, sizeof(struct ceph_handle));
 
-	constructing->key.chk_vi.ino.val = stx->stx_ino;
+	constructing->key.hhdl.chk_vi.ino.val = stx->stx_ino;
 #ifdef CEPH_NOSNAP
-	constructing->key.chk_vi.snapid.val = stx->stx_dev;
+	constructing->key.hhdl.chk_vi.snapid.val = stx->stx_dev;
 #endif /* CEPH_NOSNAP */
-	constructing->key.chk_fscid = export->fscid;
+	constructing->key.hhdl.chk_fscid = export->fscid;
+	constructing->key.export_id = export->export.export_id;
 	constructing->i = i;
 	constructing->up_ops = export->export.up_ops;
 

src/FSAL/FSAL_CEPH/internal.h

@@ -100,17 +100,22 @@ struct ceph_state_fd {
  * The 'private' Ceph FSAL handle
  */
 
-struct ceph_handle_key {
+struct ceph_host_handle {
 	vinodeno_t	chk_vi;
 	int64_t		chk_fscid;
 } __attribute__ ((__packed__));
 
+struct ceph_handle_key {
+	struct ceph_host_handle hhdl;
+	uint16_t export_id;
+};
+
 struct ceph_handle {
 	struct fsal_obj_handle handle;	/*< The public handle */
 	struct ceph_fd fd;
 	struct Inode *i;	/*< The Ceph inode */
 	const struct fsal_up_vector *up_ops;	/*< Upcall operations */
-	/*< The first export this handle belongs to */
+	/*< The export this handle belongs to */
 	struct ceph_export *export;
 	struct ceph_handle_key key;
 	struct fsal_share share;

src/FSAL/FSAL_CEPH/main.c

@@ -301,7 +301,7 @@ static int select_filesystem(struct ceph_export *export)
 static void ino_release_cb(void *handle, vinodeno_t vino)
 {
 	struct ceph_export *export = handle;
-	struct ceph_handle_key key;
+	struct ceph_host_handle key;
 	struct gsh_buffdesc fh_desc;
 
 	LogDebug(COMPONENT_FSAL,