mds: retry rdlock_path_pin_ref if needs to fwd request to auth

qa/suites/fs/libcephfs/overrides/multimds.yaml

@@ -0,0 +1,4 @@
+overrides:
+  ceph:
+    cephfs:
+      max_mds: 2

qa/workunits/libcephfs/test.sh

@@ -7,5 +7,6 @@ ceph_test_libcephfs_lazyio
 ceph_test_libcephfs_newops
 ceph_test_libcephfs_suidsgid
 ceph_test_libcephfs_snapdiff
+ceph_test_libcephfs_forwardauth
 
 exit 0

src/client/Client.cc

@@ -1697,6 +1697,12 @@ mds_rank_t Client::choose_target_mds(MetaRequest *req, Inode** phash_diri)
     goto out;
   }
 
+  if (unlikely(cct->_conf->client_debug_force_send_request_to_rank0)) {
+    mds = 0;
+    ldout(cct, 10) << __func__ << " force sending requests to mds.0" << dendl;
+    goto out;
+  }
+
   if (cct->_conf->client_use_random_mds)
     goto random_mds;
 

src/common/options/mds-client.yaml.in

@@ -245,6 +245,14 @@ options:
   fmt_desc: If set to ``true``, clients read data directly from OSDs instead
     of using a local page cache.
   with_legacy: true
+- name: client_debug_force_send_request_to_rank0
+  type: bool
+  level: dev
+  default: false
+  services:
+  - mds_client
+  fmt_desc: If set to ``true``, clients will always send client requests to rank 0.
+  with_legacy: true
 - name: client_debug_inject_tick_delay
   type: secs
   level: dev

src/mds/Server.cc

@@ -4456,8 +4456,12 @@ void Server::handle_client_open(const MDRequestRef& mdr)
   if (!cur)
     return;
 
-  if (cur->is_frozen() || cur->state_test(CInode::STATE_EXPORTINGCAPS)) {
-    ceph_assert(!need_auth);
+  // In openc() the 'PATH_LOCKED' will be set and if the file exists then it will
+  // call open() dirrectly, but when the create requests need be handled in auth
+  // MDS but the current MDS is not auth because of the inode is under exporting
+  // and the auth is changed, we need to retry rdlock_path_pin_ref() and forward
+  // the requests to the auth.
+  if (need_auth || cur->is_frozen() || cur->state_test(CInode::STATE_EXPORTINGCAPS)) {
     mdr->locking_state &= ~(MutationImpl::PATH_LOCKED | MutationImpl::ALL_LOCKED);
     CInode *cur = rdlock_path_pin_ref(mdr, true);
     if (!cur)

src/test/libcephfs/CMakeLists.txt

@@ -22,6 +22,20 @@ if(WITH_LIBCEPHFS)
   install(TARGETS ceph_test_libcephfs
     DESTINATION ${CMAKE_INSTALL_BINDIR})
 
+  add_executable(ceph_test_libcephfs_forwardauth
+    main.cc
+    forwardauth.cc
+  )
+  target_link_libraries(ceph_test_libcephfs_forwardauth
+    ceph-common
+    cephfs
+    ${UNITTEST_LIBS}
+    ${EXTRALIBS}
+    ${CMAKE_DL_LIBS}
+    )
+  install(TARGETS ceph_test_libcephfs_forwardauth
+    DESTINATION ${CMAKE_INSTALL_BINDIR})
+
   add_executable(ceph_test_libcephfs_snapdiff
     main.cc
     snapdiff.cc

src/test/libcephfs/forwardauth.cc

@@ -0,0 +1,95 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2021 Red Hat Inc.
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "gtest/gtest-spi.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock-more-matchers.h"
+#include "include/compat.h"
+#include "include/cephfs/libcephfs.h"
+#include "include/fs_types.h"
+#include "mds/mdstypes.h"
+#include "include/stat.h"
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <string.h>
+
+#ifdef __linux__
+#include <limits.h>
+#include <sys/xattr.h>
+#endif
+
+#include <fmt/format.h>
+#include <map>
+#include <vector>
+#include <thread>
+#include <regex>
+#include <string>
+
+using ::testing::AnyOf;
+using ::testing::Gt;
+using ::testing::Eq;
+using namespace std;
+
+/*
+ * Test this with different ceph versions
+ */
+
+TEST(LibCephFS, RequestForwardAuth)
+{
+  struct ceph_mount_info *cmount;
+  ASSERT_EQ(0, ceph_create(&cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_read_file(cmount, NULL));
+  ASSERT_EQ(0, ceph_conf_parse_env(cmount, NULL));
+  ASSERT_EQ(0, ceph_mount(cmount, "/"));
+
+  char test_path[1024];
+  sprintf(test_path, "test_forwardauth_dir_%d", getpid());
+
+  ASSERT_EQ(0, ceph_mkdirs(cmount, test_path, 0777));
+
+  {
+    char value[1024] = "";
+    int r = -1;
+
+    ASSERT_EQ(0, ceph_setxattr(cmount, test_path, "ceph.dir.pin", (void*)"1", 1, XATTR_CREATE));
+
+    r = ceph_getxattr(cmount, test_path, "ceph.dir.pin", (void*)value, sizeof(value));
+    ASSERT_GT(r, 0);
+    ASSERT_LT(r, sizeof value);
+    ASSERT_STREQ("1", value);
+  }
+
+  ASSERT_EQ(ceph_conf_set(cmount, "client_debug_force_send_request_to_rank0", "true"), 0);
+
+  char file_path[1024];
+  sprintf(file_path, "/%s/forward_auth_file_%d", test_path, getpid());
+
+  // create the file first and then close it
+  int fd = ceph_open(cmount, file_path, O_CREAT, 0644);
+  ASSERT_LT(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  // open it and it will forward this client request to mds.1
+  fd = ceph_open(cmount, file_path, O_CREAT|O_TRUNC|O_WRONLY, 0644);
+  ASSERT_LT(0, fd);
+  ASSERT_EQ(0, ceph_close(cmount, fd));
+
+  ASSERT_EQ(0, ceph_unlink(cmount, file_path));
+  ASSERT_EQ(0, ceph_rmdir(cmount, test_path));
+
+  ceph_shutdown(cmount);
+}