NC | pass noobaa args to gpfs

Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com>

config.js

@@ -869,6 +869,10 @@ config.MASTER_KEYS_EXEC_MAX_RETRIES = 3;
 config.NC_DISABLE_ACCESS_CHECK = false;
 config.NC_DISABLE_SCHEMA_CHECK = false;
 
+////////// GPFS //////////
+config.GPFS_DOWN_DELAY = 1000;
+
+
 //Quota
 config.QUOTA_LOW_THRESHOLD = 80;
 config.QUOTA_MAX_OBJECTS = Number.MAX_SAFE_INTEGER;
@@ -1063,7 +1067,8 @@ function load_nsfs_nc_config() {
         const merged_config = _.merge(shared_config, node_config || {});
 
         Object.keys(merged_config).forEach(function(key) {
-            if (key === 'NOOBAA_LOG_LEVEL' || key === 'UV_THREADPOOL_SIZE' || key === 'GPFS_DL_PATH') {
+            const config_to_env = ['NOOBAA_LOG_LEVEL', 'UV_THREADPOOL_SIZE', 'GPFS_DL_PATH'];
+            if (config_to_env.includes(key)) {
                 process.env[key] = merged_config[key];
                 return;
             }

docs/dev_guide/NonContainerizedDeveloperCustomizations.md

@@ -476,6 +476,25 @@ Example:
 3. systemctl restart noobaa_nsfs
 ## Config.json example 
 ```
+
+## 24. GPFS down delay -
+**Description -** Set delay (ms) of GPFS syscalls when daemon is down, to hold client replies during failover, service restart required.
+
+**Configuration Key -** GPFS_DOWN_DELAY
+
+**Type -** number
+
+**Default -** 1000
+
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"GPFS_DOWN_DELAY": "60000"
+3. systemctl restart noobaa
+```
+
 > cat /path/to/config_dir/config.json
 {
     "ENDPOINT_PORT": 80,

src/cmd/nsfs.js

@@ -4,6 +4,7 @@
 
 require('../util/dotenv').load();
 require('aws-sdk/lib/maintenance_mode_message').suppress = true;
+const { cluster } = require('../util/fork_utils');
 
 const dbg = require('../util/debug_module')(__filename);
 if (!dbg.get_process_name()) dbg.set_process_name('nsfs');
@@ -384,6 +385,15 @@ async function main(argv = minimist(process.argv.slice(2))) {
 function verify_gpfslib() {
     if (!nb_native().fs.gpfs) {
         new NoobaaEvent(NoobaaEvent.GPFSLIB_MISSING).create_event(undefined, { gpfs_dl_path: process.env.GPFS_DL_PATH }, undefined);
+        return;
+    }
+    if (cluster.isPrimary) {
+        const gpfs_noobaa_args = {
+            version: 0,
+            delay: Number(config.GPFS_DOWN_DELAY),
+            flags: 0
+        };
+        nb_native().fs.gpfs.register_gpfs_noobaa(gpfs_noobaa_args);
     }
 }
 

src/native/common.gypi

@@ -36,7 +36,7 @@
                     '-std=gnu99', # c99 -> gnu99 to allow asm()
                 ],
                 'cflags_cc': [
-                    '-std=c++17'
+                    '-std=c++20'
                 ],
                 'ldflags': [
                     '-lrt', # librt
@@ -65,7 +65,7 @@
                     # Reference - http://help.apple.com/xcode/mac/8.0/#/itcaec37c2a6
                     'MACOSX_DEPLOYMENT_TARGET': '12.4',
                     'CLANG_CXX_LIBRARY': 'libc++',
-                    'CLANG_CXX_LANGUAGE_STANDARD': 'c++17', # -std=c++17
+                    'CLANG_CXX_LANGUAGE_STANDARD': 'c++20', # -std=c++20
                     'GCC_C_LANGUAGE_STANDARD': 'c99', # -std=c99
                     'GCC_ENABLE_CPP_EXCEPTIONS': 'YES',
                 },

src/native/fs/fs_napi.cpp

@@ -158,6 +158,7 @@ DBG_INIT(0);
 typedef std::map<std::string, std::string> XattrMap;
 
 const char* gpfs_dl_path = std::getenv("GPFS_DL_PATH");
+
 int gpfs_lib_file_exists = -1;
 
 static int (*dlsym_gpfs_fcntl)(gpfs_file_t file, void* arg) = 0;
@@ -171,6 +172,17 @@ static int (*dlsym_gpfs_linkatif)(
 static int (*dlsym_gpfs_unlinkat)(
     gpfs_file_t fileDesc, const char* path, gpfs_file_t fd) = 0;
 
+static int (*dlsym_gpfs_ganesha)(
+    int op, void *oarg) = 0;
+
+struct gpfs_ganesha_noobaa_arg
+{
+    int noobaa_version;
+    int noobaa_delay;
+    int noobaa_flags;
+};
+#define OPENHANDLE_REGISTER_NOOBAA 157
+
 static const int DIO_BUFFER_MEMALIGN = 4096;
 
 static void
@@ -2115,6 +2127,30 @@ set_debug_level(const Napi::CallbackInfo& info)
     return info.Env().Undefined();
 }
 
+/**
+ * register noobaa args to GPFS
+ */
+static Napi::Value
+register_gpfs_noobaa(const Napi::CallbackInfo& info)
+{
+    Napi::Object params = info[0].As<Napi::Object>();
+    struct gpfs_ganesha_noobaa_arg args = {
+        .noobaa_version = params.Get("version").ToNumber(),
+        .noobaa_delay = params.Get("delay").ToNumber(),
+        .noobaa_flags = params.Get("flags").ToNumber(),
+    };
+    LOG("FS::GPFS gpfs_ganesha_noobaa_arg=" << DVAL(args.noobaa_version) << DVAL(args.noobaa_delay) << DVAL(args.noobaa_flags));
+
+    if (dlsym_gpfs_ganesha(OPENHANDLE_REGISTER_NOOBAA, &args)) {
+        if (errno == EOPNOTSUPP) {
+            LOG("Warning: register with libgpfs gpfs_ganesha returned EOPNOTSUPP" );
+        } else {
+            PANIC("Error: register with libgpfs gpfs_ganesha failed");
+        }
+    }
+    return info.Env().Undefined();
+}
+
 /**
  * Allocate memory aligned buffer for direct IO.
  */
@@ -2163,15 +2199,18 @@ fs_napi(Napi::Env env, Napi::Object exports)
                 PANIC("Error: %s\n"
                     << uv_dlerror(lib));
             }
+            if (uv_dlsym(lib, "gpfs_ganesha", (void**)&dlsym_gpfs_ganesha)) {
+                PANIC("Error: %s\n"
+                    << uv_dlerror(lib));
+            }
             if (sizeof(struct gpfsRequest_t) != 256) {
                 PANIC("The gpfs get extended attributes is of wrong size" << sizeof(struct gpfsRequest_t));
             }
+
             auto gpfs = Napi::Object::New(env);
-            // for now we export an (empty) object, which can be checked to indicate that
+            gpfs["register_gpfs_noobaa"] = Napi::Function::New(env, register_gpfs_noobaa);
+            // we export the gpfs object, which can be checked to indicate that
             // gpfs lib was loaded and its api's can be used.
-            // e.g: gpfs["version"] = Napi::String::New(env,  gpfs_get_version());
-            // e.g: gpfs["foo"] = Napi::Function::New(env, api<Foo>);
-            // gpfs.Freeze();
             exports_fs["gpfs"] = gpfs;
         }
     } else {

src/sdk/nb.d.ts

@@ -981,7 +981,9 @@ interface NativeFS {
     O_DIRECT?: number;
     O_TMPFILE?: number;
 
-    gpfs?: object;
+    gpfs?: {
+        register_gpfs_noobaa(gpfs_noobaa_args: GPFSNooBaaArgs);
+    };
 }
 
 interface NativeFile {
@@ -1014,6 +1016,12 @@ interface NativeFSContext {
     disable_ctime_check?: boolean;
 }
 
+type GPFSNooBaaArgs = {
+    version: number;
+    delay: number;
+    flags: number;
+};
+
 type NativeFSXattr = { [key: string]: string };
 type NativeFSStats = fs.Stats & {
     atimeNsBigint: bigint;

src/server/system_services/schemas/nsfs_config_schema.js

@@ -46,6 +46,10 @@ const nsfs_node_config_schema = {
             type: 'string',
             doc: 'indicates the location of the gpfs library file, service restart required, usually should be set to /usr/lib64/libgpfs.so'
         },
+        GPFS_DOWN_DELAY: {
+            type: 'number',
+            doc: 'delay (ms) of GPFS syscalls when daemon is down, to hold client replies during failover, service restart required'
+        },
         NSFS_NC_STORAGE_BACKEND: {
             $ref: 'common_api#/definitions/fs_backend',
             doc: 'indicates the global storage backend type, service restart required'

src/util/fork_utils.js

@@ -134,3 +134,4 @@ function _set_op_stats(op_name, stats) {
 }
 
 exports.start_workers = start_workers;
+exports.cluster = cluster;