Skip to content

Commit

Permalink
updated algorithm to handle mid month shift in logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@justinlevi
justinlevi committed Oct 27, 2020
1 parent bcffc58 commit b9b50d5
Show file tree
Hide file tree
Showing 8 changed files with 13,854 additions and 2,208 deletions.
107 changes: 84 additions & 23 deletions services/api/src/models/environment.test.ts
View file
Original file line number Diff line number Diff line change
@@ -1,68 +1,127 @@
import { EnvironmentModel } from "./environment";

import aLegacy from "./mockHitDataLegacy-project-a.json";
import aLegacyMissingData from "./mockHitDataLegacy-project-a-missing-data.json";
import bNew from "./mockHitDataNew-project-b.json";
import bNewMissingData from "./mockHitDataNew-project-b-missing-data.json";
import cLegacyPartial from "./mockHitDataLegacy-project-c.json";
import cNewPartial from "./mockHitDataNew-project-c.json"



// const esClient = {
// search: () => { hits: { total: {value: 999 } }}
// }
// const environmentModel = EnvironmentModel({esClient});




import esClient from '../clients/esClient';


const zeroOutBuckets = (mockResult) => {
return {
...mockResult,
hits: { total: {value : 0 }},
aggregations: {
hourly : {
buckets : mockResult.aggregations.hourly.buckets.map(bucket => ({
...bucket,
count: { value: 0 }
}))
},
average : {
value : 0
}
}
}
}

// Unit Under Test
describe('Environment Data', () => {

describe('Hits', () => {
// scenarios and expectation
it('When using the legacy logging system, then the hits should be 4461658', async () => {
it('When using the legacy logging system, with all Sept hourly buckets total set to 10, then the hits should be 7200 #legacy #logging', async () => {
// Arrange
const environmentModel = EnvironmentModel({esClient});
const openshiftProjectName = "test-legacy-prod";
const month = "2020-09"
const projectName = "test-legacy"
esClient.search = async (obj) => {
// new query request - return empty object
if(obj.index === `router-logs-${projectName}-_-*`){
return zeroOutBuckets(aLegacy);
}
return aLegacy;
}

// Act
const hits = await environmentModel.environmentHitsMonthByEnvironmentId(projectName, openshiftProjectName, month);

// Assert
expect(hits.total).toBe(7200);
});

// scenarios and expectation
it('When using the legacy logging system, with missing data (20 Sept hourly buckets set to zero) (720 total buckets set to 10), then the hits should be 7200 #legacy #logging #missing-data', async () => {
// Arrange
const environmentModel = EnvironmentModel({esClient});
const openshiftProjectName = "test-prod";
const month = "2020-10"
const projectName = "test"
const openshiftProjectName = "test-legacy-prod";
const month = "2020-09"
const projectName = "test-legacy"
esClient.search = async (obj) => {
// new query request - return empty object
if(obj.index === `router-logs-${projectName}-_-*`){
return zeroOutBuckets(aLegacyMissingData);
}
return aLegacyMissingData;
}

// Act
const hits = await environmentModel.environmentHitsMonthByEnvironmentId(projectName, openshiftProjectName, month);

// Assert
expect(hits.total).toBe(4461658);
expect(hits.total).toBe(7200);
});

// scenarios and expectation
it('When using the new logging system, then the hits should be ', async () => {
it('When using the new logging system, with all Sept 2020 hourly buckets total set to 2, then the hits should be 1440 #new #logging', async () => {
// Arrange
const environmentModel = EnvironmentModel({esClient});
const openshiftProjectName = "test-prod";
const month = "2020-09"
const projectName = "test"
esClient.search = async (obj) => {
// legacy query request - return empty object
if(obj.index === `router-logs-${openshiftProjectName}-*`){
return zeroOutBuckets(bNew);
}
return bNew;
}

// Act
const hits = await environmentModel.environmentHitsMonthByEnvironmentId(projectName, openshiftProjectName, month);

// Assert
expect(hits.total).toBe(1440);
});

// scenarios and expectation
it('When using the new logging system, with missing data, (20 Sept hourly buckets set to zero) (720 total buckets set to 2), then the hits should be 1440 #new #logging #missing-data', async () => {
// Arrange
const environmentModel = EnvironmentModel({esClient});
const openshiftProjectName = "test-prod";
const month = "2020-10"
const month = "2020-09"
const projectName = "test"
esClient.search = async (obj) => {
// legacy query request - return empty object
if(obj.index === `router-logs-${openshiftProjectName}-*`){
return zeroOutBuckets(bNewMissingData);
}
return bNewMissingData;
}

// Act
const hits = await environmentModel.environmentHitsMonthByEnvironmentId(projectName, openshiftProjectName, month);

// Assert
expect(hits.total).toBe(138934);
expect(hits.total).toBe(1440);
});



// scenarios and expectation
it('When a project uses both the new and legacy logging system, then the hits should be ', async () => {
it('When a project uses both the new and legacy logging system, then the hits should be 7200 #partial #new #legacy #logging', async () => {
// Arrange
const openshiftProjectName = "test-partial-prod";
esClient.search = async (obj) => {
Expand All @@ -75,16 +134,18 @@ describe('Environment Data', () => {
return {}
}
const environmentModel = EnvironmentModel({esClient});
const month = "2020-10"
const month = "2020-09"
const projectName = "test"

// Act
const hits = await environmentModel.environmentHitsMonthByEnvironmentId(projectName, openshiftProjectName, month);

// Assert
expect(hits.total).toBe(4461658);
expect(hits.total).toBe(7200);
});



});

}); // End Unit Under Test
45 changes: 28 additions & 17 deletions services/api/src/models/environment.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -420,6 +420,18 @@ export const EnvironmentModel = (clients) => {

var total = 0;

const legacyBuckets = legacyResult && legacyResult.aggregations && legacyResult.aggregations.hourly && legacyResult.aggregations.hourly.buckets ? legacyResult.aggregations.hourly.buckets : 0;
const legacyResultCount = legacyResult && legacyResult.aggregations && legacyResult.aggregations.hourly && legacyResult.aggregations.hourly.buckets && legacyResult.aggregations.hourly.buckets.length ? legacyResult.aggregations.hourly.buckets.length : 0;
const legacyAvg = legacyResult && legacyResult.aggregations && legacyResult.aggregations.average && legacyResult.aggregations.average.value ? legacyResult.aggregations.average.value : 0;

const newBuckets = newResult && newResult.aggregations && newResult.aggregations.hourly && newResult.aggregations.hourly.buckets ? newResult.aggregations.hourly.buckets : 0;
const newResultCount = newResult && newResult.aggregations && newResult.aggregations.hourly && newResult.aggregations.hourly.buckets && newResult.aggregations.hourly.buckets.length ? newResult.aggregations.hourly.buckets.length : 0;
const newAvg = newResult && newResult.aggregations && newResult.aggregations.average && newResult.aggregations.average.value ? newResult.aggregations.average.value : 0;

if (legacyResultCount !== newResultCount){
throw new Error("Legacy logging buckets count does not equal New logging buckets count. Something is fishy...");
}

/*
foreach hourlybucket (#both result buckets should have the exact same amount of buckets)
add to total
Expand All @@ -431,26 +443,25 @@ export const EnvironmentModel = (clients) => {
--> if both are 0, use newResult.average
*/

const legacyBuckets = legacyResult.aggregations.hourly.buckets;
const legacyResultCount = legacyResult.aggregations.hourly.buckets.length;

const newBuckets = newResult.aggregations.hourly.buckets;
const newResultCount = newResult.aggregations.hourly.buckets.length;

if (legacyResultCount !== newResultCount){
throw new Error("Legacy logging buckets count does not equal New logging buckets count. Something is fishy...");
}

for (let i = 0; i < legacyResultCount; i++) {

if (newResultCount !== 0 && newBuckets[i].count && newBuckets[i].count.value !== 0){
// We have new logging data, use this for total
total += newBuckets[i].count.value;
}else if (legacyResultCount !== 0 && legacyBuckets[i].count && legacyBuckets[i].count.value !== 0){
// We have legacy data
total += legacyBuckets[i].count.value;
}else{
// Both legacy and new logging buckets are zero, meaning we have missing data, use the avg
if(newAvg !== 0){
total += newAvg;
}else if (legacyAvg !== 0){
total += legacyAvg;
}else{
total += newAvg;
}
}
}

// loop through all hourly sum counts
// if the sum count is empty, this means we have missing data and we use the overall average instead.
result.aggregations.hourly.buckets.forEach(bucket => {
total += (bucket.count.value === 0 ? parseInt(result.aggregations.average.value) : bucket.count.value);
});

return { total };

} catch (e) {
Expand Down

0 comments on commit b9b50d5

Please sign in to comment.