Skip to content

Commit

Permalink
livemetrics: add lastSendSucceeded to prevent permanent retries (#504)
Browse files Browse the repository at this point in the history 
* add lastSendSucceeded to prevent permanent retries

* add qps time constants, remove static class state

* fix request averages being NaN
  • Loading branch information
@markwolff
markwolff committed Apr 5, 2019
1 parent 4433089 commit bef5907
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 20 deletions.
4 changes: 2 additions & 2 deletions AutoCollection/Performance.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ class AutoCollectPerformance {
var intervalFailedRequests = (requests.totalFailedRequestCount - lastRequests.totalFailedRequestCount) || 0;
var elapsedMs = requests.time - lastRequests.time;
var elapsedSeconds = elapsedMs / 1000;
var averageRequestExecutionTime = AutoCollectPerformance._intervalRequestExecutionTime / intervalRequests;
var averageRequestExecutionTime = (AutoCollectPerformance._intervalRequestExecutionTime / intervalRequests) || 0; // default to 0 in case no requests in this interval
AutoCollectPerformance._intervalRequestExecutionTime = 0; // reset

if (elapsedMs > 0) {
Expand Down Expand Up @@ -285,7 +285,7 @@ class AutoCollectPerformance {
var intervalFailedDependencies = (dependencies.totalFailedDependencyCount - lastDependencies.totalFailedDependencyCount) || 0;
var elapsedMs = dependencies.time - lastDependencies.time;
var elapsedSeconds = elapsedMs / 1000;
var averageDependencyExecutionTime = AutoCollectPerformance._intervalDependencyExecutionTime / intervalDependencies;
var averageDependencyExecutionTime = (AutoCollectPerformance._intervalDependencyExecutionTime / intervalDependencies) || 0;
AutoCollectPerformance._intervalDependencyExecutionTime = 0; // reset

if (elapsedMs > 0) {
Expand Down
9 changes: 5 additions & 4 deletions Library/QuickPulseSender.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,17 @@ class QuickPulseSender {
this._config = config;
}

public ping(envelope: Contracts.EnvelopeQuickPulse, done: (shouldPOST: boolean) => void): void {
public ping(envelope: Contracts.EnvelopeQuickPulse, done: (shouldPOST: boolean, res?: http.IncomingMessage) => void): void {
this._submitData(envelope, done, "ping");
}

public post(envelope: Contracts.EnvelopeQuickPulse, done: (shouldPOST: boolean, res: http.IncomingMessage) => void): void {
public post(envelope: Contracts.EnvelopeQuickPulse, done: (shouldPOST: boolean, res?: http.IncomingMessage) => void): void {

// Important: When POSTing data, envelope must be an array
this._submitData([envelope], done, "post");
}

private _submitData(envelope: Contracts.EnvelopeQuickPulse | Contracts.EnvelopeQuickPulse[], done: (shouldPOST: boolean, res: http.IncomingMessage) => void, postOrPing: "post" | "ping"): void {
private _submitData(envelope: Contracts.EnvelopeQuickPulse | Contracts.EnvelopeQuickPulse[], done: (shouldPOST: boolean, res?: http.IncomingMessage) => void, postOrPing: "post" | "ping"): void {
const payload = JSON.stringify(envelope);
var options = {
[AutoCollectHttpDependencies.disableCollectionRequestOption]: true,
Expand All @@ -52,7 +52,8 @@ class QuickPulseSender {
req.on("error", (error: Error) => {
// Unable to contact qps endpoint.
// Do nothing for now.
Logging.warn("Unable to contact qps endpoint", error);
Logging.warn("Unable to contact qps endpoint, dropping this Live Metrics packet", error);
done(false); // Stop POSTing QPS data
});

req.write(payload);
Expand Down
35 changes: 23 additions & 12 deletions Library/QuickPulseStateManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,23 @@ import Context = require("./Context");
import * as http from "http";
import * as Contracts from "../Declarations/Contracts";


/** State Container for sending to the QuickPulse Service */
class QuickPulseStateManager {
public config: Config;
public context: Context;

private static _isCollectingData: boolean = false;
private static MAX_POST_WAIT_TIME = 20000;
private static MAX_PING_WAIT_TIME = 60000;
private static FALLBACK_INTERVAL = 60000;
private static PING_INTERVAL = 5000;
private static POST_INTERVAL = 1000;

private _isCollectingData: boolean = false;
private _sender: QuickPulseSender;
private _isEnabled: boolean;
private _lastSuccessTime: number = Date.now();
private _lastSendSucceeded: boolean = true;
private _handle: NodeJS.Timer;
private _metrics: {[name: string]: Contracts.MetricQuickPulse} = {};
private _documents: Contracts.DocumentQuickPulse[] = [];
Expand Down Expand Up @@ -117,21 +124,22 @@ class QuickPulseStateManager {
this._resetQuickPulseBuffer();

// Send it to QuickPulseService, if collecting
if (QuickPulseStateManager._isCollectingData) {
if (this._isCollectingData) {
this._post(envelope);
} else {
this._ping(envelope);
}

let currentTimeout = QuickPulseStateManager._isCollectingData ? 1000 : 5000;
if (QuickPulseStateManager._isCollectingData && Date.now() - this._lastSuccessTime >= 20000) {
let currentTimeout = this._isCollectingData ? QuickPulseStateManager.POST_INTERVAL : QuickPulseStateManager.PING_INTERVAL;
if (this._isCollectingData && Date.now() - this._lastSuccessTime >= QuickPulseStateManager.MAX_POST_WAIT_TIME && !this._lastSendSucceeded) {
// Haven't posted successfully in 20 seconds, so wait 60 seconds and ping
QuickPulseStateManager._isCollectingData = false;
currentTimeout = 60000;
} else if (!QuickPulseStateManager._isCollectingData && Date.now() - this._lastSuccessTime >= 60000) {
this._isCollectingData = false;
currentTimeout = QuickPulseStateManager.FALLBACK_INTERVAL;
} else if (!this._isCollectingData && Date.now() - this._lastSuccessTime >= QuickPulseStateManager.MAX_PING_WAIT_TIME && !this._lastSendSucceeded) {
// Haven't pinged successfully in 60 seconds, so wait another 60 seconds
currentTimeout = 60000;
currentTimeout = QuickPulseStateManager.FALLBACK_INTERVAL;
}
this._lastSendSucceeded = null;
this._handle = <any>setTimeout(this._goQuickPulse.bind(this), currentTimeout);
this._handle.unref(); // Don't block apps from terminating
}
Expand All @@ -144,15 +152,18 @@ class QuickPulseStateManager {
this._sender.post(envelope, this._quickPulseDone.bind(this));
}

private _quickPulseDone(shouldPOST: boolean, res: http.IncomingMessage): void {
if (QuickPulseStateManager._isCollectingData !== shouldPOST) {
private _quickPulseDone(shouldPOST: boolean, res?: http.IncomingMessage): void {
if (this._isCollectingData !== shouldPOST) {
Logging.info("Live Metrics sending data", shouldPOST);
this.enableCollectors(shouldPOST);
}
QuickPulseStateManager._isCollectingData = shouldPOST;
this._isCollectingData = shouldPOST;

if (res.statusCode < 300 && res.statusCode >= 200) {
if (res && res.statusCode < 300 && res.statusCode >= 200) {
this._lastSuccessTime = Date.now();
this._lastSendSucceeded = true;
} else {
this._lastSendSucceeded = false;
}
}

Expand Down
4 changes: 2 additions & 2 deletions Tests/Library/QuickPulseStateManager.tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ describe("Library/QuickPulseStateManager", () => {
assert.equal(qps.config.instrumentationKey, "ikey");
assert.ok(qps.context);
assert.equal(qps["_isEnabled"], false);
assert.equal(QuickPulseClient["_isCollectingData"], false);
assert.equal(qps["_isCollectingData"], false);
assert.ok(qps["_sender"]);
assert.ok(Object.keys(qps["_metrics"]).length === 0);
assert.ok(qps["_documents"].length === 0);
Expand Down Expand Up @@ -105,7 +105,7 @@ describe("Library/QuickPulseStateManager", () => {
assert.ok(pingStub.notCalled);
assert.ok(postStub.notCalled);

QuickPulseClient["_isCollectingData"] = true;
qps["_isCollectingData"] = true;
qps.enable(true)

assert.ok(postStub.calledOnce);
Expand Down

0 comments on commit bef5907

Please sign in to comment.