Skip to content

Commit

Permalink
feat: Backoff based on HTTP statuses (#537)
Browse files Browse the repository at this point in the history 
New behaviour:
 * Will emit error event and stop posting metrics/polling features on 401,403 and 404
 * Will emit warn event and back-off to 2,3,4,5,6,7,8,9,10 * normal poll/metrics interval on 429,500,502,503 and 504.
  • Loading branch information
@chriswk
chriswk committed Nov 21, 2023
1 parent 8aefb0e commit 748d793
Show file tree
Hide file tree
Showing 8 changed files with 545 additions and 126 deletions.
2 changes: 1 addition & 1 deletion src/events.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export enum UnleashEvents {
CountVariant = 'countVariant',
Sent = 'sent',
Registered = 'registered',
Impression = 'impression'
Impression = 'impression',
}

export interface ImpressionEvent {
Expand Down
90 changes: 67 additions & 23 deletions src/metrics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ interface MetricsData {
}

interface RegistrationData {
appName: string;
appName: string;
instanceId: string;
sdkVersion: string;
strategies: string[];
started: Date;
interval: number
started: Date;
interval: number;
}

export default class Metrics extends EventEmitter {
Expand All @@ -61,6 +61,8 @@ export default class Metrics extends EventEmitter {

private metricsJitter: number;

private failures: number = 0;

private disabled: boolean;

private url: string;
Expand Down Expand Up @@ -111,21 +113,39 @@ export default class Metrics extends EventEmitter {
return getAppliedJitter(this.metricsJitter);
}

getFailures(): number {
return this.failures;
}

getInterval(): number {
if(this.metricsInterval === 0) {
return 0;
} else {
return this.metricsInterval +
(this.failures * this.metricsInterval) +
this.getAppliedJitter();
}

}

private startTimer(): void {
if (this.disabled) {
if (this.disabled || this.getInterval() === 0) {
return;
}
this.timer = setTimeout(() => {
this.sendMetrics();
}, this.metricsInterval + this.getAppliedJitter());
this.timer = setTimeout(
() => {
this.sendMetrics();
},
this.getInterval(),
);

if (process.env.NODE_ENV !== 'test' && typeof this.timer.unref === 'function') {
this.timer.unref();
}
}

start(): void {
if (typeof this.metricsInterval === 'number' && this.metricsInterval > 0) {
if (this.metricsInterval > 0) {
this.startTimer();
this.registerInstance();
}
Expand Down Expand Up @@ -170,6 +190,19 @@ export default class Metrics extends EventEmitter {
return true;
}

configurationError(url: string, statusCode: number) {
this.emit(UnleashEvents.Warn, `${url} returning ${statusCode}, stopping metrics`);
this.metricsInterval = 0;
this.stop();
}

backoff(url: string, statusCode: number): void {
this.failures = Math.min(10, this.failures + 1);
// eslint-disable-next-line max-len
this.emit(UnleashEvents.Warn, `${url} returning ${statusCode}. Backing off to ${this.failures} times normal interval`);
this.startTimer();
}

async sendMetrics(): Promise<void> {
if (this.disabled) {
return;
Expand All @@ -194,16 +227,22 @@ export default class Metrics extends EventEmitter {
timeout: this.timeout,
httpOptions: this.httpOptions,
});
this.startTimer();
if (res.status === 404) {
this.emit(UnleashEvents.Warn, `${url} returning 404, stopping metrics`);
this.stop();
}
if (!res.ok) {
if (res.status === 404 || res.status === 403 || res.status == 401) {
this.configurationError(url, res.status);
} else if (
res.status === 429 ||
res.status === 500 ||
res.status === 502 ||
res.status === 503 ||
res.status === 504
) {
this.backoff(url, res.status);
}
this.restoreBucket(payload.bucket);
this.emit(UnleashEvents.Warn, `${url} returning ${res.status}`, await res.text());
} else {
this.emit(UnleashEvents.Sent, payload);
this.reduceBackoff();
}
} catch (err) {
this.restoreBucket(payload.bucket);
Expand All @@ -212,6 +251,11 @@ export default class Metrics extends EventEmitter {
}
}

reduceBackoff(): void {
this.failures = Math.max(0, this.failures - 1);
this.startTimer();
}

assertBucket(name: string): void {
if (this.disabled) {
return;
Expand Down Expand Up @@ -243,7 +287,7 @@ export default class Metrics extends EventEmitter {
}

private increaseCounter(name: string, enabled: boolean, inc = 1): void {
if(inc === 0) {
if (inc === 0) {
return;
}
this.assertBucket(name);
Expand All @@ -252,8 +296,8 @@ export default class Metrics extends EventEmitter {

private increaseVariantCounter(name: string, variantName: string, inc = 1): void {
this.assertBucket(name);
if(this.bucket.toggles[name].variants[variantName]) {
this.bucket.toggles[name].variants[variantName]+=inc
if (this.bucket.toggles[name].variants[variantName]) {
this.bucket.toggles[name].variants[variantName] += inc;
} else {
this.bucket.toggles[name].variants[variantName] = inc;
}
Expand All @@ -276,7 +320,7 @@ export default class Metrics extends EventEmitter {
}

createMetricsData(): MetricsData {
const bucket = {...this.bucket, stop: new Date()};
const bucket = { ...this.bucket, stop: new Date() };
this.resetBucket();
return {
appName: this.appName,
Expand All @@ -286,20 +330,20 @@ export default class Metrics extends EventEmitter {
}

private restoreBucket(bucket: Bucket): void {
if(this.disabled) {
if (this.disabled) {
return;
}
this.bucket.start = bucket.start;

const { toggles } = bucket;
Object.keys(toggles).forEach(toggleName => {
const toggle = toggles[toggleName];
Object.keys(toggles).forEach((toggleName) => {
const toggle = toggles[toggleName];
this.increaseCounter(toggleName, true, toggle.yes);
this.increaseCounter(toggleName, false, toggle.no);

Object.keys(toggle.variants).forEach(variant => {
Object.keys(toggle.variants).forEach((variant) => {
this.increaseVariantCounter(toggleName, variant, toggle.variants[variant]);
})
});
});
}

Expand Down
95 changes: 88 additions & 7 deletions src/repository/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ export default class Repository extends EventEmitter implements EventEmitter {

private headers?: CustomHeaders;

private failures: number = 0;

private customHeadersFunction?: CustomHeadersFunction;

private timeout?: number;
Expand Down Expand Up @@ -241,11 +243,92 @@ Message: ${err.message}`,
return obj;
}

getFailures(): number {
return this.failures;
}

nextFetch(): number {
return this.refreshInterval + this.failures * this.refreshInterval;
}

private backoff(): number {
this.failures = Math.min(this.failures + 1, 10);
return this.nextFetch();
}

private countSuccess(): number {
this.failures = Math.max(this.failures - 1, 0);
return this.nextFetch();
}

// Emits correct error message based on what failed,
// and returns 0 as the next fetch interval (stop polling)
private configurationError(url: string, statusCode: number): number {
this.failures += 1;
if (statusCode === 404) {
this.emit(
UnleashEvents.Error,
new Error(
// eslint-disable-next-line max-len
`${url} responded NOT_FOUND (404) which means your API url most likely needs correction. Stopping refresh of toggles`,
),
);
} else if (statusCode === 401 || statusCode === 403) {
this.emit(
UnleashEvents.Error,
new Error(
// eslint-disable-next-line max-len
`${url} responded ${statusCode} which means your API key is not allowed to connect. Stopping refresh of toggles`,
),
);
}
return 0;
}

// We got a status code we know what to do with, so will log correct message
// and return the new interval.
private recoverableError(url: string, statusCode: number): number {
let nextFetch = this.backoff();
if (statusCode === 429) {
this.emit(
UnleashEvents.Warn,
// eslint-disable-next-line max-len
`${url} responded TOO_MANY_CONNECTIONS (429). Backing off`,
);
} else if (statusCode === 500 ||
statusCode === 502 ||
statusCode === 503 ||
statusCode === 504) {
this.emit(
UnleashEvents.Warn,
`${url} responded ${statusCode}. Backing off`,
);
}
return nextFetch;
}

private handleErrorCases(url: string, statusCode: number): number {
if (statusCode === 401 || statusCode === 403 || statusCode === 404) {
return this.configurationError(url, statusCode);
} else if (
statusCode === 429 ||
statusCode === 500 ||
statusCode === 502 ||
statusCode === 503 ||
statusCode === 504
) {
return this.recoverableError(url, statusCode);
} else {
const error = new Error(`Response was not statusCode 2XX, but was ${statusCode}`);
this.emit(UnleashEvents.Error, error);
return this.refreshInterval;
}
}

async fetch(): Promise<void> {
if (this.stopped || !(this.refreshInterval > 0)) {
return;
}

let nextFetch = this.refreshInterval;
try {
let mergedTags;
Expand All @@ -257,7 +340,6 @@ Message: ${err.message}`,
const headers = this.customHeadersFunction
? await this.customHeadersFunction()
: this.headers;

const res = await get({
url,
etag: this.etag,
Expand All @@ -268,14 +350,11 @@ Message: ${err.message}`,
httpOptions: this.httpOptions,
supportedSpecVersion: SUPPORTED_SPEC_VERSION,
});

if (res.status === 304) {
// No new data
this.emit(UnleashEvents.Unchanged);
} else if (!res.ok) {
const error = new Error(`Response was not statusCode 2XX, but was ${res.status}`);
this.emit(UnleashEvents.Error, error);
} else {
} else if (res.ok) {
nextFetch = this.countSuccess();
try {
const data: ClientFeaturesResponse = await res.json();
if (res.headers.get('etag') !== null) {
Expand All @@ -287,6 +366,8 @@ Message: ${err.message}`,
} catch (err) {
this.emit(UnleashEvents.Error, err);
}
} else {
nextFetch = this.handleErrorCases(url, res.status);
}
} catch (err) {
const e = err as { code: string };
Expand Down

0 comments on commit 748d793

Please sign in to comment.