Skip to content

Commit

Permalink
Merge pull request #8094 from romayalon/romy-remove-syslog-from-health
Browse files Browse the repository at this point in the history 
NC | Health | Remove rsyslog/syslog_ng checks
  • Loading branch information
@romayalon
romayalon committed May 29, 2024
2 parents 9fdaa9c + 7a7b3df commit 958ad06
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 127 deletions.
40 changes: 6 additions & 34 deletions docs/non_containerized_NSFS.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,8 +237,8 @@ NOTE - health script execution requires root permissions.
"status": "NOTOK",
"memory": "88.6M",
"error": {
"error_code": "RSYSLOG_SERVICE_FAILED",
"error_message": "RSYSLOG service is not started properly, Please verify the service with status command."
"error_code": "NOOBAA_SERVICE_FAILED",
"error_message": "NooBaa service is not started properly, Please verify the service with status command."
},
"checks": {
"services": [
Expand All @@ -247,12 +247,6 @@ NOTE - health script execution requires root permissions.
"service_status": "active",
"pid": "1204",
"error_type": "PERSISTENT"
},
{
"name": "rsyslog",
"service_status": "inactive",
"pid": "0",
"error_type": "PERSISTENT"
}
],
"endpoint": {
Expand Down Expand Up @@ -317,9 +311,9 @@ NOTE - health script execution requires root permissions.

`error_message`: Message explaining the issue with the health script.

`service_status`: NooBaa systemd status. Check for noobaa/rsyslog service up and running.
`service_status`: NooBaa systemd status. Check for noobaa service up and running.

`pid`: NooBaa/Rsyslog systemd process id.
`pid`: NooBaa systemd process id.

`endpoint_response`: Noobaa endpoint web service response code.

Expand All @@ -341,12 +335,6 @@ In this health output, `bucket2`'s storage path is invalid and the directory men

Account without `new_buckets_path` and `allow_bucket_creation` value is `false` then it's considered a valid account, But if the `allow_bucket_creation` is true `new_buckets_path` is empty, in that case account is invalid.

### Optional status checks

#### 1. `check_syslog_ng`
`check_syslog_ng` flag will add syslogng to the health status check. Health final status will depend on the syslogng status.
Health script will check whether the syslogng service is active or not and its PID is a valid value. Syslogng status will get added along with NooBaa and rsyslog.

### Health Error Codes
These are the error codes populated in the health output if the system is facing some issues. If any of these error codes are present in health status then the overall status will be in `NOTOK` state.
#### 1. `NOOBAA_SERVICE_FAILED`
Expand All @@ -365,24 +353,8 @@ If the NooBaa service is not started, start the service
systemctl enable noobaa.service
systemctl start noobaa.service
```
#### 2. `RSYSLOG_SERVICE_FAILED`
#### Reasons
- Rsysog service is not started properly.
- Stopped Rsyslog service is not removed.

#### Resolutions
- Verify the Rsyslog service is running by checking the status and logs command.
```
systemctl status rsyslog.service
journalctl -xeu rsyslog.service
```
If the rsyslog is not started, start the service
```
systemctl enable rsyslog.service
systemctl start rsyslog.service
```

#### 3. `NOOBAA_ENDPOINT_FORK_MISSING`
#### 2. `NOOBAA_ENDPOINT_FORK_MISSING`
#### Reasons
- One or more endpoint fork is not started properly.
- Number of workers running is less than the configured `forks` value.
Expand All @@ -394,7 +366,7 @@ systemctl status rsyslog.service
journalctl -xeu rsyslog.service
```

#### 4. `NOOBAA_ENDPOINT_FAILED`
#### 3. `NOOBAA_ENDPOINT_FAILED`
#### Reasons
- NooBaa endpoint process is not running and Its not able to respond to any requests.

Expand Down
45 changes: 6 additions & 39 deletions src/cmd/health.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ Flags:
--https_port (optional) Set the S3 endpoint listening HTTPS port to serve. (default config.ENDPOINT_SSL_PORT)
--all_account_details (optional) Set a flag for returning all account details.
--all_bucket_details (optional) Set a flag for returning all bucket details.
--check_syslog_ng (optional) Set a flag for considering syslog-ng in health check.
`;

function print_usage() {
Expand All @@ -49,17 +48,12 @@ function print_usage() {

const HOSTNAME = 'localhost';
const NOOBAA_SERVICE = 'noobaa';
const RSYSLOG_SERVICE = 'rsyslog';
const SYSLOG_NG_SERVICE = 'syslog-ng';

const health_errors = {
NOOBAA_SERVICE_FAILED: {
error_code: 'NOOBAA_SERVICE_FAILED',
error_message: 'NooBaa service is not started properly, Please verify the service with status command.',
},
RSYSLOG_SERVICE_FAILED: {
error_code: 'RSYSLOG_SERVICE_FAILED',
error_message: 'RSYSLOG service is not started properly, Please verify the service with status command.',
},
NOOBAA_ENDPOINT_FAILED: {
error_code: 'NOOBAA_ENDPOINT_FAILED',
error_message: 'S3 endpoint process is not running. Restart the endpoint process.',
Expand Down Expand Up @@ -125,12 +119,10 @@ class NSFSHealth {
this.config_root = options.config_root;
this.all_account_details = options.all_account_details;
this.all_bucket_details = options.all_bucket_details;
this.check_syslog_ng = options.check_syslog_ng;
}
async nc_nsfs_health() {
let endpoint_state;
let memory;
let syslog_ng;
const { service_status, pid } = await this.get_service_state(NOOBAA_SERVICE);
if (pid !== '0') {
endpoint_state = await this.get_endpoint_response();
Expand All @@ -139,20 +131,12 @@ class NSFSHealth {
let bucket_details;
let account_details;
const response_code = endpoint_state ? endpoint_state.response.response_code : 'NOT_RUNNING';
const rsyslog = await this.get_service_state(RSYSLOG_SERVICE);
let service_health = 'OK';
let syslog_ng_health = 'OK';
if (this.check_syslog_ng) {
syslog_ng = await this.get_service_state(SYSLOG_NG_SERVICE);
if (syslog_ng.service_status !== 'active' || syslog_ng.pid === '0') {
syslog_ng_health = 'NOTOK';
}
}
if (syslog_ng_health === 'NOTOK' || service_status !== 'active' || pid === '0' || response_code !== 'RUNNING' ||
rsyslog.service_status !== 'active' || rsyslog.pid === '0') {

if (service_status !== 'active' || pid === '0' || response_code !== 'RUNNING') {
service_health = 'NOTOK';
}
const error_code = await this.get_error_code(service_status, pid, rsyslog.service_status, response_code);
const error_code = await this.get_error_code(service_status, pid, response_code);
if (this.all_bucket_details) bucket_details = await this.get_bucket_status(this.config_root);
if (this.all_account_details) account_details = await this.get_account_status(this.config_root);
const health = {
Expand All @@ -166,12 +150,6 @@ class NSFSHealth {
service_status: service_status,
pid: pid,
error_type: health_errors_tyes.PERSISTENT,
},
{
name: RSYSLOG_SERVICE,
service_status: rsyslog.service_status,
pid: rsyslog.pid,
error_type: health_errors_tyes.PERSISTENT,
}
],
endpoint: {
Expand All @@ -190,14 +168,6 @@ class NSFSHealth {
}
}
};
if (this.check_syslog_ng) {
health.checks.services.push({
name: SYSLOG_NG_SERVICE,
service_status: syslog_ng.service_status,
pid: syslog_ng.pid,
error_type: health_errors_tyes.PERSISTENT,
});
}
if (!this.all_account_details) delete health.checks.accounts_status;
if (!this.all_bucket_details) delete health.checks.buckets_status;
return health;
Expand Down Expand Up @@ -225,11 +195,9 @@ class NSFSHealth {
return endpoint_state;
}

async get_error_code(nsfs_status, pid, rsyslog_status, endpoint_response_code) {
async get_error_code(nsfs_status, pid, endpoint_response_code) {
if (nsfs_status !== 'active' || pid === '0') {
return health_errors.NOOBAA_SERVICE_FAILED;
} else if (rsyslog_status !== 'active') {
return health_errors.RSYSLOG_SERVICE_FAILED;
} else if (endpoint_response_code === 'NOT_RUNNING') {
return health_errors.NOOBAA_ENDPOINT_FAILED;
} else if (endpoint_response_code === 'MISSING_FORKS') {
Expand Down Expand Up @@ -420,9 +388,8 @@ async function main(argv = minimist(process.argv.slice(2))) {
const deployment_type = argv.deployment_type || 'nc';
const all_account_details = argv.all_account_details || false;
const all_bucket_details = argv.all_bucket_details || false;
const check_syslog_ng = argv.check_syslog_ng || false;
if (deployment_type === 'nc') {
const health = new NSFSHealth({ https_port, config_root, all_account_details, all_bucket_details, check_syslog_ng });
const health = new NSFSHealth({ https_port, config_root, all_account_details, all_bucket_details });
const health_status = await health.nc_nsfs_health();
process.stdout.write(JSON.stringify(health_status) + '\n', () => {
process.exit(0);
Expand Down
54 changes: 0 additions & 54 deletions src/test/unit_tests/test_nc_nsfs_health.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,19 +119,6 @@ mocha.describe('nsfs nc health', function() {
assert.strictEqual(health_status.error.error_code, 'NOOBAA_SERVICE_FAILED');
});

mocha.it('NooBaa rsyslog service is inactive', async function() {
Health.get_service_state.restore();
Health.get_endpoint_response.restore();
const get_service_state = sinon.stub(Health, "get_service_state");
get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 }))
.onSecondCall().returns(Promise.resolve({ service_status: 'inactive', pid: 0 }));
const get_endpoint_response = sinon.stub(Health, "get_endpoint_response");
get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}}));
const health_status = await Health.nc_nsfs_health();
assert.strictEqual(health_status.status, 'NOTOK');
assert.strictEqual(health_status.error.error_code, 'RSYSLOG_SERVICE_FAILED');
});

mocha.it('NooBaa endpoint return error response is inactive', async function() {
Health.get_service_state.restore();
Health.get_endpoint_response.restore();
Expand Down Expand Up @@ -365,47 +352,6 @@ mocha.describe('nsfs nc health', function() {
assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 2);
await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_invalid.name + '.json'));
});

mocha.it('Health command with syslog_ng optional check is true', async function() {
Health.get_service_state.restore();
Health.get_endpoint_response.restore();
Health.all_account_details = true;
Health.all_bucket_details = false;
Health.check_syslog_ng = true;
const get_service_state = sinon.stub(Health, "get_service_state");
get_service_state.onCall(0).returns(Promise.resolve({ service_status: 'active', pid: 1000 }))
.onCall(1)
.returns(Promise.resolve({ service_status: 'active', pid: 2000 }))
.onCall(2)
.returns(Promise.resolve({ service_status: 'active', pid: 3000 }));
const get_endpoint_response = sinon.stub(Health, "get_endpoint_response");
get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}}));
const health_status = await Health.nc_nsfs_health();
assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 2);
assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1);
assert.strictEqual(health_status.checks.services[2].name, 'syslog-ng');
assert.strictEqual(health_status.checks.services[2].service_status, 'active');
});

mocha.it('Health command with syslog_ng optional check is false', async function() {
Health.get_service_state.restore();
Health.get_endpoint_response.restore();
Health.all_account_details = true;
Health.all_bucket_details = false;
Health.check_syslog_ng = false;
const get_service_state = sinon.stub(Health, "get_service_state");
get_service_state.onCall(0).returns(Promise.resolve({ service_status: 'active', pid: 1000 }))
.onCall(1)
.returns(Promise.resolve({ service_status: 'active', pid: 2000 }))
.onCall(2)
.returns(Promise.resolve({ service_status: 'active', pid: 3000 }));
const get_endpoint_response = sinon.stub(Health, "get_endpoint_response");
get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}}));
const health_status = await Health.nc_nsfs_health();
assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 2);
assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1);
assert.strictEqual(health_status.checks.services.length, 2);
});
});
});

Expand Down

0 comments on commit 958ad06

Please sign in to comment.