rgw: add CORTX Motr SAL implementation

CMakeLists.txt

@@ -394,6 +394,7 @@ option(WITH_RADOSGW_AMQP_ENDPOINT "Rados Gateway's pubsub support for AMQP push
 option(WITH_RADOSGW_KAFKA_ENDPOINT "Rados Gateway's pubsub support for Kafka push endpoint" ON)
 option(WITH_RADOSGW_LUA_PACKAGES "Rados Gateway's support for dynamically adding lua packagess" ON)
 option(WITH_RADOSGW_DBSTORE "DBStore backend for Rados Gateway" ON)
+option(WITH_RADOSGW_MOTR "CORTX-Motr backend for Rados Gateway" OFF)
 option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" OFF)
 
 if(WITH_RADOSGW)

install-deps.sh

@@ -306,6 +306,8 @@ else
     [ $WITH_JAEGER ] && with_jaeger=true || with_jaeger=false
     [ $WITH_ZBD ] && with_zbd=true || with_zbd=false
     [ $WITH_PMEM ] && with_pmem=true || with_pmem=false
+    [ $WITH_RADOSGW_MOTR ] && with_rgw_motr=true || with_rgw_motr=false
+    motr_pkgs_url='https://github.com/Seagate/cortx-motr/releases/download/2.0.0-rgw'
     source /etc/os-release
     case "$ID" in
     debian|ubuntu|devuan|elementary|softiron)
@@ -362,15 +364,34 @@ else
 	      --tool="apt-get -y --no-install-recommends $backports" $control || exit 1
 	$SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove ceph-build-deps
 	if [ "$control" != "debian/control" ] ; then rm $control; fi
+
+        # for rgw motr backend build checks
+        $for_make_check || $with_rgw_motr && {
+            deb_arch=$(dpkg --print-architecture)
+            motr_pkg="cortx-motr_2.0.0.git3252d623_$deb_arch.deb"
+            motr_dev_pkg="cortx-motr-dev_2.0.0.git3252d623_$deb_arch.deb"
+            $SUDO curl -sL -o/var/cache/apt/archives/$motr_pkg $motr_pkgs_url/$motr_pkg
+            $SUDO curl -sL -o/var/cache/apt/archives/$motr_dev_pkg $motr_pkgs_url/$motr_dev_pkg
+            # For some reason libfabric pkg is not available in arm64 version
+            # of Ubuntu 20.04 (Focal Fossa), so we borrow it from more recent
+            # versions for now.
+            if [[ "$deb_arch" == 'arm64' ]]; then
+                lf_pkg='libfabric1_1.11.0-2_arm64.deb'
+                $SUDO curl -sL -o/var/cache/apt/archives/$lf_pkg http://ports.ubuntu.com/pool/universe/libf/libfabric/$lf_pkg
+                $SUDO apt-get install -y /var/cache/apt/archives/$lf_pkg
+            fi
+            $SUDO apt-get install -y /var/cache/apt/archives/{$motr_pkg,$motr_dev_pkg}
+            $SUDO apt-get install -y libisal-dev
+        }
         ;;
-    centos|fedora|rhel|ol|virtuozzo)
+    rocky|centos|fedora|rhel|ol|virtuozzo)
         builddepcmd="dnf -y builddep --allowerasing"
         echo "Using dnf to install dependencies"
         case "$ID" in
             fedora)
                 $SUDO dnf install -y dnf-utils
                 ;;
-            centos|rhel|ol|virtuozzo)
+            rocky|centos|rhel|ol|virtuozzo)
                 MAJOR_VERSION="$(echo $VERSION_ID | cut -d. -f1)"
                 $SUDO dnf install -y dnf-utils
                 rpm --quiet --query epel-release || \
@@ -397,6 +418,13 @@ else
         [ ${PIPESTATUS[0]} -ne 0 ] && exit 1
         IGNORE_YUM_BUILDEP_ERRORS="ValueError: SELinux policy is not managed or store cannot be accessed."
         sed "/$IGNORE_YUM_BUILDEP_ERRORS/d" $DIR/yum-builddep.out | grep -qi "error:" && exit 1
+        # for rgw motr backend build checks
+        $for_make_check || $with_rgw_motr && {
+            $SUDO dnf install -y \
+                  "$motr_pkgs_url/isa-l-2.30.0-1.el7.${ARCH}.rpm" \
+                  "$motr_pkgs_url/cortx-motr-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm" \
+                  "$motr_pkgs_url/cortx-motr-devel-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm"
+        }
         ;;
     opensuse*|suse|sles)
         echo "Using zypper to install dependencies"

run-make-check.sh

@@ -90,6 +90,7 @@ function main() {
     cmake_opts+=" -DWITH_CEPHFS_SHELL=ON"
     cmake_opts+=" -DWITH_GRAFANA=ON"
     cmake_opts+=" -DWITH_SPDK=ON"
+    cmake_opts+=" -DWITH_RADOSGW_MOTR=ON"
     if [ $WITH_SEASTAR ]; then
         cmake_opts+=" -DWITH_SEASTAR=ON"
     fi

src/common/options/rgw.yaml.in

@@ -3241,13 +3241,53 @@ options:
   type: str
   level: advanced
   desc: experimental Option to set backend store type
-  long_desc: defaults to rados. Other valid values are dbstore(experimental).
+  long_desc: defaults to rados. Other valid values are dbstore and motr (both experimental).
   default: rados
   services:
   - rgw
   enum_values:
   - rados
   - dbstore
+  - motr
+- name: motr_profile_fid
+  type: str
+  level: advanced
+  desc: experimental Option to set Motr profile fid
+  long_desc: example value 0x7000000000000001:0x4f
+  default: 0x7000000000000001:0x0
+  services:
+  - rgw
+- name: motr_my_fid
+  type: str
+  level: advanced
+  desc: experimental Option to set my Motr fid
+  long_desc: example value 0x7200000000000001:0x29
+  default: 0x7200000000000001:0x0
+  services:
+  - rgw
+- name: motr_my_endpoint
+  type: str
+  level: advanced
+  desc: experimental Option to set my Motr endpoint address
+  long_desc: example value 192.168.180.182@tcp:12345:4:1
+  default: 192.168.180.182@tcp:12345:4:1
+  services:
+  - rgw
+- name: motr_ha_endpoint
+  type: str
+  level: advanced
+  desc: experimental Option to set Motr HA agent endpoint address
+  long_desc: example value 192.168.180.182@tcp:12345:1:1
+  default: 192.168.180.182@tcp:12345:1:1
+  services:
+  - rgw
+- name: motr_tracing_enabled
+  type: bool
+  level: advanced
+  desc: Set to true when Motr client debugging is needed
+  default: false
+  services:
+  - rgw
 - name: rgw_luarocks_location
   type: str
   level: advanced

src/include/config-h.in.cmake

@@ -351,6 +351,9 @@
 /* Backend dbstore for Rados Gateway */
 #cmakedefine WITH_RADOSGW_DBSTORE
 
+/* Backend CORTX-Motr for Rados Gateway */
+#cmakedefine WITH_RADOSGW_MOTR
+
 /* Defined if std::map::merge() is supported */
 #cmakedefine HAVE_STDLIB_MAP_SPLICING
 

src/rgw/CMakeLists.txt

@@ -176,6 +176,9 @@ if(WITH_RADOSGW_DBSTORE)
   add_subdirectory(store/dbstore)
   list(APPEND librgw_common_srcs rgw_sal_dbstore.cc)
 endif()
+if(WITH_RADOSGW_MOTR)
+  list(APPEND librgw_common_srcs rgw_sal_motr.cc)
+endif()
 if(WITH_JAEGER)
   list(APPEND librgw_common_srcs rgw_tracer.cc)
 endif()
@@ -258,6 +261,14 @@ if(WITH_RADOSGW_DBSTORE)
   target_link_libraries(rgw_common PRIVATE global dbstore)
 endif()
 
+if(WITH_RADOSGW_MOTR)
+  target_include_directories(rgw_common PRIVATE "/usr/include/motr")
+  target_compile_options(rgw_common PRIVATE "-Wno-attributes")
+  target_compile_definitions(rgw_common
+    PRIVATE "M0_EXTERN=extern" "M0_INTERNAL=")
+  target_link_libraries(rgw_common PRIVATE motr motr-helpers)
+endif()
+
 set(rgw_a_srcs
   rgw_auth_keystone.cc
   rgw_client_io.cc

src/rgw/rgw_main.cc

@@ -360,6 +360,12 @@ int radosgw_Main(int argc, const char **argv)
   }
 #endif
 
+#ifdef WITH_RADOSGW_MOTR
+  if (config_store == "motr") {
+    rgw_store = "motr";
+  }
+#endif
+
   rgw::sal::Store* store =
     StoreManager::get_storage(&dp, g_ceph_context,
 				 rgw_store,

src/rgw/rgw_sal.cc

@@ -29,13 +29,21 @@
 #include "rgw_sal_dbstore.h"
 #endif
 
+#ifdef WITH_RADOSGW_MOTR
+#include "rgw_sal_motr.h"
+#endif
+
+
 #define dout_subsys ceph_subsys_rgw
 
 extern "C" {
 extern rgw::sal::Store* newStore(void);
 #ifdef WITH_RADOSGW_DBSTORE
 extern rgw::sal::Store* newDBStore(CephContext *cct);
 #endif
+#ifdef WITH_RADOSGW_MOTR
+extern rgw::sal::Store* newMotrStore(CephContext *cct);
+#endif
 }
 
 rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_cache, bool use_gc)
@@ -76,8 +84,8 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d
     return store;
   }
 
-  if (svc.compare("dbstore") == 0) {
 #ifdef WITH_RADOSGW_DBSTORE
+  if (svc.compare("dbstore") == 0) {
     rgw::sal::Store* store = newDBStore(cct);
 
     if ((*(rgw::sal::DBStore*)store).set_run_lc_thread(use_lc_thread)
@@ -99,8 +107,50 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d
       ldpp_dout(dpp, 0) << "ERROR: failed inserting testid user in dbstore error r=" << r << dendl;
     }
     return store;
+  }
 #endif
+
+#ifdef WITH_RADOSGW_MOTR
+  if (svc.compare("motr") == 0) {
+    rgw::sal::Store* store = newMotrStore(cct);
+    if (store == nullptr) {
+      ldpp_dout(dpp, 0) << "newMotrStore() failed!" << dendl;
+      return store;
+    }
+    ((rgw::sal::MotrStore *)store)->init_metadata_cache(dpp, cct);
+
+    /* XXX: temporary - create testid user */
+    rgw_user testid_user("tenant", "tester", "ns");
+    std::unique_ptr<rgw::sal::User> user = store->get_user(testid_user);
+    user->get_info().user_id = testid_user;
+    user->get_info().display_name = "Motr Explorer";
+    user->get_info().user_email = "tester@seagate.com";
+    RGWAccessKey k1("0555b35654ad1656d804", "h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==");
+    user->get_info().access_keys["0555b35654ad1656d804"] = k1;
+
+    ldpp_dout(dpp, 20) << "Store testid and user for Motr. User = " << user->get_info().user_id.id << dendl;
+    int rc = user->store_user(dpp, null_yield, true);
+    if (rc < 0) {
+      ldpp_dout(dpp, 0) << "ERROR: failed to store testid user ar Motr: rc=" << rc << dendl;
+    }
+
+    // Read user info and compare.
+    rgw_user ruser("", "tester", "");
+    std::unique_ptr<rgw::sal::User> suser = store->get_user(ruser);
+    suser->get_info().user_id = ruser;
+    rc = suser->load_user(dpp, null_yield);
+    if (rc != 0) {
+      ldpp_dout(dpp, 0) << "ERROR: failed to load testid user from Motr: rc=" << rc << dendl;
+    } else {
+      ldpp_dout(dpp, 20) << "Read and compare user info: " << dendl;
+      ldpp_dout(dpp, 20) << "User id = " << suser->get_info().user_id.id << dendl;
+      ldpp_dout(dpp, 20) << "User display name = " << suser->get_info().display_name << dendl;
+      ldpp_dout(dpp, 20) << "User email = " << suser->get_info().user_email << dendl;
+    }
+
+    return store;
   }
+#endif
 
   return nullptr;
 }
@@ -133,6 +183,14 @@ rgw::sal::Store* StoreManager::init_raw_storage_provider(const DoutPrefixProvide
     store = nullptr;
 #endif
   }
+
+  if (svc.compare("motr") == 0) {
+#ifdef WITH_RADOSGW_MOTR
+    store = newMotrStore(cct);
+#else
+    store = nullptr;
+#endif
+  }
   return store;
 }
 

src/rgw/rgw_sal.h

@@ -126,7 +126,7 @@ class RGWGetUserStats_CB : public RefCountedObject {
  *
  * Data within RGW is stored as an Object.  Each Object is a single chunk of data, owned
  * by a single User, contained within a single Bucket.  It has metadata associated with
- * it, such as size, owner, and so in, and a set of key-value attributes that can
+ * it, such as size, owner, and so on, and a set of key-value attributes that can
  * contain anything needed by the top half.
  *
  * Data with RGW is organized into Buckets.  Each Bucket is owned by a User, and
@@ -244,7 +244,7 @@ class Store {
     Store() {}
     virtual ~Store() = default;
 
-    /** Name of this store provider (e.g., RADOS") */
+    /** Name of this store provider (e.g., "rados") */
     virtual const char* get_name() const = 0;
     /** Get cluster unique identifier */
     virtual std::string get_cluster_id(const DoutPrefixProvider* dpp,  optional_yield y) = 0;
@@ -288,7 +288,7 @@ class Store {
      /** Get a @a Notification object.  Used to communicate with non-RGW daemons, such as
       * management/tracking software */
     /** RGWOp variant */
-    virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, struct req_state* s, 
+    virtual std::unique_ptr<Notification> get_notification(rgw::sal::Object* obj, rgw::sal::Object* src_obj, struct req_state* s,
         rgw::notify::EventType event_type, const std::string* object_name=nullptr) = 0;
     /** No-req_state variant (e.g., rgwlc) */
     virtual std::unique_ptr<Notification> get_notification(
@@ -400,7 +400,7 @@ class Store {
 
     /** Get the Ceph context associated with this store.  May be removed. */
     virtual CephContext* ctx(void) = 0;
-    
+
     /** Get the location of where lua packages are installed */
     virtual const std::string& get_luarocks_path() const = 0;
     /** Set the location of where lua packages are installed */
@@ -489,6 +489,7 @@ class User {
     /** Set the attributes in attrs, leaving any other existing attrs set, and
      * write them to the backing store; a merge operation */
     virtual int merge_and_store_attrs(const DoutPrefixProvider* dpp, Attrs& new_attrs, optional_yield y) = 0;
+    /** Read the User stats from the backing Store, synchronous */
     virtual int read_stats(const DoutPrefixProvider *dpp,
                            optional_yield y, RGWStorageStats* stats,
 			   ceph::real_time* last_stats_sync = nullptr,