Skip to content

[AZI-529] improve log forwarder retry behavior #596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 2, 2022
Merged

[AZI-529] improve log forwarder retry behavior #596

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0a9f24e
[AZI-529] add retry with delay
parsons90 Jul 27, 2022
d3c8970
[AZI-529] skip retrying 4xx responses
parsons90 Jul 28, 2022
604db1c
[AZI-529] fix status code retry
parsons90 Jul 28, 2022
0a314a1
[AZI_529] change retry timeout
parsons90 Aug 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 58 additions & 39 deletions azure/activity_logs_monitoring/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

var https = require('https');

const VERSION = '0.5.6';
const VERSION = '0.5.7';

const STRING = 'string'; // example: 'some message'
const STRING_ARRAY = 'string-array'; // example: ['one message', 'two message', ...]
Expand All @@ -29,6 +29,9 @@ const DD_SERVICE = process.env.DD_SERVICE || 'azure';
const DD_SOURCE = process.env.DD_SOURCE || 'azure';
const DD_SOURCE_CATEGORY = process.env.DD_SOURCE_CATEGORY || 'azure';

const MAX_RETRIES = 4; // max number of times to retry a single http request
const RETRY_INTERVAL = 250; // amount of time (milliseconds) to wait before retrying request, doubles after every retry

/*
To scrub PII from your logs, uncomment the applicable configs below. If you'd like to scrub more than just
emails and IP addresses, add your own config to this map in the format
Expand Down Expand Up @@ -155,52 +158,70 @@ class HTTPClient {
var batches = this.batcher.batch(records);
var promises = [];
for (var i = 0; i < batches.length; i++) {
promises.push(this.sendWithRetry(batches[i]));
promises.push(this.send(batches[i]));
}
return await Promise.all(
promises.map(p => p.catch(e => this.context.log.error(e)))
);
}

sendWithRetry(record) {
return new Promise((resolve, reject) => {
return this.send(record)
.then(res => {
resolve(true);
})
.catch(err => {
this.send(record)
.then(res => {
resolve(true);
})
.catch(err => {
reject(
`unable to send request after 2 tries, err: ${err}`
);
});
});
});
isStatusCodeValid(statusCode) {
return statusCode >= 200 && statusCode <= 299;
}

shouldStatusCodeRetry(statusCode) {
// don't retry 4xx responses
return (
!this.isStatusCodeValid(statusCode) &&
(statusCode < 400 || statusCode > 499)
);
}

send(record) {
var numRetries = MAX_RETRIES;
var retryInterval = RETRY_INTERVAL;
return new Promise((resolve, reject) => {
const req = https
.request(this.httpOptions, resp => {
if (resp.statusCode < 200 || resp.statusCode > 299) {
reject(`invalid status code ${resp.statusCode}`);
} else {
resolve(true);
const sendRequest = (options, record) => {
const retryRequest = errMsg => {
if (numRetries === 0) {
return reject(errMsg);
}
})
.on('error', error => {
reject(error);
});
req.on('timeout', () => {
req.destroy();
reject(`request timed out after ${DD_REQUEST_TIMEOUT_MS}ms`);
})
req.write(this.scrubber.scrub(JSON.stringify(record)));
req.end();
this.context.log.warn(
`Unable to send request, with error: ${errMsg}. Retrying ${numRetries} more times`
);
numRetries--;
retryInterval *= 2;
setTimeout(() => {
sendRequest(options, record);
}, retryInterval);
};
const req = https
.request(options, resp => {
if (this.isStatusCodeValid(resp.statusCode)) {
resolve(true);
} else if (
this.shouldStatusCodeRetry(resp.statusCode)
) {
retryRequest(
`invalid status code ${resp.statusCode}`
);
} else {
reject(`invalid status code ${resp.statusCode}`);
}
})
.on('error', error => {
retryRequest(error.message);
})
.on('timeout', () => {
req.destroy();
retryRequest(
`request timed out after ${DD_REQUEST_TIMEOUT_MS}ms`
);
});
req.write(this.scrubber.scrub(JSON.stringify(record)));
req.end();
};
sendRequest(this.httpOptions, record);
});
}
}
Expand All @@ -219,9 +240,7 @@ class Scrubber {
);
} catch {
context.log.error(
`Regexp for rule ${name} pattern ${
settings['pattern']
} is malformed, skipping. Please update the pattern for this rule to be applied.`
`Regexp for rule ${name} pattern ${settings['pattern']} is malformed, skipping. Please update the pattern for this rule to be applied.`
);
}
}
Expand Down