Merge Stage 5.15.4 to 5.15 | Backports

.eslintrc.js

@@ -12,6 +12,7 @@ module.exports = {
     env: {
         node: true,
         es2022: true,
+        jest: true,
     },
 
     // See https://eslint.org/docs/latest/user-guide/configuring/language-options#specifying-environments

config.js

@@ -823,6 +823,7 @@ config.NSFS_LOW_FREE_SPACE_PERCENT_UNLEASH = 0.10;
 // NSFS NON CONTAINERIZED //
 ////////////////////////////
 
+config.ENDPOINT_PROCESS_TITLE = 'noobaa';
 config.NC_RELOAD_CONFIG_INTERVAL = 10 * 1000;
 config.NSFS_NC_CONF_DIR_REDIRECT_FILE = 'config_dir_redirect';
 config.NSFS_NC_DEFAULT_CONF_DIR = '/etc/noobaa.conf.d';
@@ -854,6 +855,10 @@ config.NC_MASTER_KEYS_PUT_EXECUTABLE = '';
 config.NC_MASTER_KEYS_MANAGER_REFRESH_THRESHOLD = -1; // currently we want to disable automatic refresh
 config.MASTER_KEYS_EXEC_MAX_RETRIES = 3;
 
+////////// GPFS //////////
+config.GPFS_DOWN_DELAY = 1000;
+
+
 //Quota
 config.QUOTA_LOW_THRESHOLD = 80;
 config.QUOTA_MAX_OBJECTS = Number.MAX_SAFE_INTEGER;
@@ -1048,7 +1053,8 @@ function load_nsfs_nc_config() {
         const merged_config = _.merge(shared_config, node_config || {});
 
         Object.keys(merged_config).forEach(function(key) {
-            if (key === 'NOOBAA_LOG_LEVEL' || key === 'UV_THREADPOOL_SIZE' || key === 'GPFS_DL_PATH') {
+            const config_to_env = ['NOOBAA_LOG_LEVEL', 'UV_THREADPOOL_SIZE', 'GPFS_DL_PATH'];
+            if (config_to_env.includes(key)) {
                 process.env[key] = merged_config[key];
                 return;
             }

docs/dev_guide/NonContainerizedDeveloperCustomizations.md

@@ -422,8 +422,42 @@ Example:
 3. systemctl restart noobaa_nsfs
 ```
 
+## 23. Set Endpoint process title -
+**Description -** This flag will set noobaa process title for letting GPFS to identify the noobaa endpoint processes. see issue #8039.
+
+**Configuration Key -** ENDPOINT_PROCESS_TITLE
+
+**Type -** string
+
+**Default -**  'noobaa'
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"ENDPOINT_PROCESS_TITLE": 'noobaa_60'
+3. systemctl restart noobaa_nsfs
 ## Config.json example 
 ```
+
+## 24. GPFS down delay -
+**Description -** Set delay (ms) of GPFS syscalls when daemon is down, to hold client replies during failover, service restart required.
+
+**Configuration Key -** GPFS_DOWN_DELAY
+
+**Type -** number
+
+**Default -** 1000
+
+**Steps -**
+```
+1. Open /path/to/config_dir/config.json file.
+2. Set the config key -
+Example:
+"GPFS_DOWN_DELAY": "60000"
+3. systemctl restart noobaa
+```
+
 > cat /path/to/config_dir/config.json
 {
     "ENDPOINT_PORT": 80,

docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md

@@ -46,3 +46,7 @@ Attached a table with tests that where investigated and their status (this table
 | test_object_create_bad_date_none_aws2                     | Internal Component | [438](https://github.com/ceph/s3-tests/issues/438)                    | It used to pass in the past (not related to code change in our repo) |
 | test_bucket_create_bad_authorization_invalid_aws2         | Internal Component | [438](https://github.com/ceph/s3-tests/issues/438)                    | It used to pass in the past (not related to code change in our repo) |
 | test_bucket_create_bad_date_none_aws2                     | Internal Component | [438](https://github.com/ceph/s3-tests/issues/438)                    | It used to pass in the past (not related to code change in our repo) |
+| test_get_object_ifnonematch_good                     | Internal Component |                    | It used to pass in the past (not related to code 
+change in our repo) - stopped passing between the update of commit hash 6861c3d81081a6883fb90d66cb60392e1abdf3ca to da91ad8bbf899c72199df35b69e9393c706aabee |
+| test_get_object_ifmodifiedsince_failed                     | Internal Component |                    | It used to pass in the past (not related to code 
+change in our repo) - stopped passing between the update of commit hash 6861c3d81081a6883fb90d66cb60392e1abdf3ca to da91ad8bbf899c72199df35b69e9393c706aabee |

docs/non_containerized_NSFS.md

@@ -237,8 +237,8 @@ NOTE - health script execution requires root permissions.
   "status": "NOTOK",
   "memory": "88.6M",
   "error": {
-    "error_code": "RSYSLOG_SERVICE_FAILED",
-    "error_message": "RSYSLOG service is not started properly, Please verify the service with status command."
+    "error_code": "NOOBAA_SERVICE_FAILED",
+    "error_message": "NooBaa service is not started properly, Please verify the service with status command."
   },
   "checks": {
     "services": [
@@ -247,12 +247,6 @@ NOTE - health script execution requires root permissions.
         "service_status": "active",
         "pid": "1204",
         "error_type": "PERSISTENT"
-      },
-      {
-        "name": "rsyslog",
-        "service_status": "inactive",
-        "pid": "0",
-        "error_type": "PERSISTENT"
       }
     ],
     "endpoint": {
@@ -317,9 +311,9 @@ NOTE - health script execution requires root permissions.
 
 `error_message`: Message explaining the issue with the health script.
 
-`service_status`: NooBaa systemd status. Check for noobaa/rsyslog service up and running.
+`service_status`: NooBaa systemd status. Check for noobaa service up and running.
 
-`pid`: NooBaa/Rsyslog systemd process id.
+`pid`: NooBaa systemd process id.
 
 `endpoint_response`: Noobaa endpoint web service response code.
 
@@ -341,12 +335,6 @@ In this health output, `bucket2`'s storage path is invalid and the directory men
 
 Account without `new_buckets_path` and `allow_bucket_creation` value is `false` then it's considered a valid account, But if the `allow_bucket_creation` is true `new_buckets_path` is empty, in that case account is invalid.
 
-### Optional status checks
-
-#### 1. `check_syslog_ng`
-`check_syslog_ng` flag will add syslogng to the health status check. Health final status will depend on the syslogng status.
-Health script will check whether the syslogng service is active or not and its PID is a valid value. Syslogng status will get added along with NooBaa and rsyslog.
-
 ### Health Error Codes
 These are the error codes populated in the health output if the system is facing some issues. If any of these error codes are present in health status then the overall status will be in `NOTOK` state.
 #### 1. `NOOBAA_SERVICE_FAILED`
@@ -365,24 +353,8 @@ If the NooBaa service is not started, start the service
 systemctl enable noobaa.service
 systemctl start noobaa.service
 ```
-#### 2. `RSYSLOG_SERVICE_FAILED`
-#### Reasons
-- Rsysog service is not started properly.
-- Stopped Rsyslog service is not removed.
 
-#### Resolutions
-- Verify the Rsyslog service is running by checking the status and logs command.
-```
-systemctl status rsyslog.service
-journalctl -xeu rsyslog.service
-```
-If the rsyslog is not started, start the service
-```
-systemctl enable rsyslog.service
-systemctl start rsyslog.service
-```
-
-#### 3. `NOOBAA_ENDPOINT_FORK_MISSING`
+#### 2. `NOOBAA_ENDPOINT_FORK_MISSING`
 #### Reasons
 - One or more endpoint fork is not started properly.
 - Number of workers running is less than the configured `forks` value.
@@ -394,7 +366,7 @@ systemctl status rsyslog.service
 journalctl -xeu rsyslog.service
 ```
 
-#### 4. `NOOBAA_ENDPOINT_FAILED`
+#### 3. `NOOBAA_ENDPOINT_FAILED`
 #### Reasons
 - NooBaa endpoint process is not running and Its not able to respond to any requests.
 
@@ -612,7 +584,9 @@ Noobaa logs are pushed to `var/log/noobaa.log` and the log is rotated and compre
 
 Verify the rsyslog and logrotate rpm configuration is complete by checking the files `etc/rsyslog.d/noobaa_syslog.conf` for rsyslog and `etc/logrotate.d/noobaa/logrotate_noobaa.conf` for logrotate.These files contain the noobaa specific configuration for rsyslog and logrotate.
 
-Rotate the logs manually.
+Logrotate configuration is set up under `/etc/logrotate.d/noobaa/`. In order to trigger rotation when log files reache size threshold, each log file (`/var/log/noobaa.log` and `/var/log/noobaa_events.log`) uses its own rsyslog `outchannel` that triggers logrotate once file size reaches a limit (105MB). Logrotate is also typically called from cron on daily schedule.
+
+To rotate the logs manually run.
 
 ```
 logrotate /etc/logrotate.d/noobaa/logrotate_noobaa.conf

package.json

@@ -1,6 +1,6 @@
 {
   "name": "noobaa-core",
-  "version": "5.15.3",
+  "version": "5.15.4",
   "license": "SEE LICENSE IN LICENSE",
   "description": "",
   "homepage": "https://github.com/noobaa/noobaa-core",

src/cmd/health.js

@@ -37,7 +37,6 @@ Flags:
     --https_port                      (optional)                             Set the S3 endpoint listening HTTPS port to serve. (default config.ENDPOINT_SSL_PORT)
     --all_account_details             (optional)                             Set a flag for returning all account details.
     --all_bucket_details              (optional)                             Set a flag for returning all bucket details.
-    --check_syslog_ng                 (optional)                             Set a flag for considering syslog-ng in health check.
 `;
 
 function print_usage() {
@@ -49,17 +48,12 @@ function print_usage() {
 
 const HOSTNAME = 'localhost';
 const NOOBAA_SERVICE = 'noobaa';
-const RSYSLOG_SERVICE = 'rsyslog';
-const SYSLOG_NG_SERVICE = 'syslog-ng';
+
 const health_errors = {
     NOOBAA_SERVICE_FAILED: {
         error_code: 'NOOBAA_SERVICE_FAILED',
         error_message: 'NooBaa service is not started properly, Please verify the service with status command.',
     },
-    RSYSLOG_SERVICE_FAILED: {
-        error_code: 'RSYSLOG_SERVICE_FAILED',
-        error_message: 'RSYSLOG service is not started properly, Please verify the service with status command.',
-    },
     NOOBAA_ENDPOINT_FAILED: {
         error_code: 'NOOBAA_ENDPOINT_FAILED',
         error_message: 'S3 endpoint process is not running. Restart the endpoint process.',
@@ -125,12 +119,10 @@ class NSFSHealth {
         this.config_root = options.config_root;
         this.all_account_details = options.all_account_details;
         this.all_bucket_details = options.all_bucket_details;
-        this.check_syslog_ng = options.check_syslog_ng;
     }
     async nc_nsfs_health() {
         let endpoint_state;
         let memory;
-        let syslog_ng;
         const { service_status, pid } = await this.get_service_state(NOOBAA_SERVICE);
         if (pid !== '0') {
             endpoint_state = await this.get_endpoint_response();
@@ -139,20 +131,12 @@ class NSFSHealth {
         let bucket_details;
         let account_details;
         const response_code = endpoint_state ? endpoint_state.response.response_code : 'NOT_RUNNING';
-        const rsyslog = await this.get_service_state(RSYSLOG_SERVICE);
         let service_health = 'OK';
-        let syslog_ng_health = 'OK';
-        if (this.check_syslog_ng) {
-            syslog_ng = await this.get_service_state(SYSLOG_NG_SERVICE);
-            if (syslog_ng.service_status !== 'active' || syslog_ng.pid === '0') {
-                syslog_ng_health = 'NOTOK';
-            }
-        }
-        if (syslog_ng_health === 'NOTOK' || service_status !== 'active' || pid === '0' || response_code !== 'RUNNING' ||
-            rsyslog.service_status !== 'active' || rsyslog.pid === '0') {
+
+        if (service_status !== 'active' || pid === '0' || response_code !== 'RUNNING') {
             service_health = 'NOTOK';
         }
-        const error_code = await this.get_error_code(service_status, pid, rsyslog.service_status, response_code);
+        const error_code = await this.get_error_code(service_status, pid, response_code);
         if (this.all_bucket_details) bucket_details = await this.get_bucket_status(this.config_root);
         if (this.all_account_details) account_details = await this.get_account_status(this.config_root);
         const health = {
@@ -166,12 +150,6 @@ class NSFSHealth {
                         service_status: service_status,
                         pid: pid,
                         error_type: health_errors_tyes.PERSISTENT,
-                    },
-                    {
-                        name: RSYSLOG_SERVICE,
-                        service_status: rsyslog.service_status,
-                        pid: rsyslog.pid,
-                        error_type: health_errors_tyes.PERSISTENT,
                     }
                 ],
                 endpoint: {
@@ -190,14 +168,6 @@ class NSFSHealth {
                 }
             }
         };
-        if (this.check_syslog_ng) {
-            health.checks.services.push({
-                name: SYSLOG_NG_SERVICE,
-                service_status: syslog_ng.service_status,
-                pid: syslog_ng.pid,
-                error_type: health_errors_tyes.PERSISTENT,
-            });
-        }
         if (!this.all_account_details) delete health.checks.accounts_status;
         if (!this.all_bucket_details) delete health.checks.buckets_status;
         return health;
@@ -225,11 +195,9 @@ class NSFSHealth {
         return endpoint_state;
     }
 
-    async get_error_code(nsfs_status, pid, rsyslog_status, endpoint_response_code) {
+    async get_error_code(nsfs_status, pid, endpoint_response_code) {
         if (nsfs_status !== 'active' || pid === '0') {
             return health_errors.NOOBAA_SERVICE_FAILED;
-        } else if (rsyslog_status !== 'active') {
-            return health_errors.RSYSLOG_SERVICE_FAILED;
         } else if (endpoint_response_code === 'NOT_RUNNING') {
             return health_errors.NOOBAA_ENDPOINT_FAILED;
         } else if (endpoint_response_code === 'MISSING_FORKS') {
@@ -420,9 +388,8 @@ async function main(argv = minimist(process.argv.slice(2))) {
         const deployment_type = argv.deployment_type || 'nc';
         const all_account_details = argv.all_account_details || false;
         const all_bucket_details = argv.all_bucket_details || false;
-        const check_syslog_ng = argv.check_syslog_ng || false;
         if (deployment_type === 'nc') {
-            const health = new NSFSHealth({ https_port, config_root, all_account_details, all_bucket_details, check_syslog_ng });
+            const health = new NSFSHealth({ https_port, config_root, all_account_details, all_bucket_details });
             const health_status = await health.nc_nsfs_health();
             process.stdout.write(JSON.stringify(health_status) + '\n', () => {
                 process.exit(0);