Skip to content

Commit

Permalink
wip(perf): backend completed fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@tabarra
tabarra committed May 23, 2024
1 parent e5b1108 commit 16b21bf
Show file tree
Hide file tree
Showing 6 changed files with 81 additions and 55 deletions.
77 changes: 42 additions & 35 deletions core/components/PerformanceCollector/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,36 +8,15 @@ import type { LogNodeHeapEventType, SSFileType, SSLogDataType, SSLogType, SSPerf
import { diffPerfs, fetchFxsMemory, fetchRawPerfData, perfCountsToHist } from './perfUtils';
import { optimizeStatsLog } from './statsLogOptimizer';
import { convars } from '@core/globalData';
import { ValuesType } from 'utility-types';
import bytes from 'bytes';
import { ZodError } from 'zod';
import { PERF_DATA_BUCKET_COUNT, PERF_DATA_INITIAL_RESOLUTION, PERF_DATA_MIN_TICKS } from './statsConfigs';
const console = consoleFactory(modulename);


//Consts
const minutesMs = 60 * 1000;
const hoursMs = 60 * minutesMs;
const megabyte = 1024 * 1024;


/**
* Configs
*/
const STATS_DATA_FILE_VERSION = 1;
const STATS_DATA_FILE_NAME = 'statsData.json';
export const PERF_DATA_BUCKET_COUNT = 15;
export const PERF_DATA_MIN_TICKS = 2000; //less than that and the data is not reliable
export const PERF_DATA_INITIAL_RESOLUTION = 5 * minutesMs;
export const STATS_RESOLUTION_TABLE = [
//00~12h = 5min = 12/h = 144 snaps
//12~24h = 15min = 4/h = 48 snaps
//24~96h = 30min = 2/h = 144 snaps
{ maxAge: 12 * hoursMs, resolution: PERF_DATA_INITIAL_RESOLUTION },
{ maxAge: 24 * hoursMs, resolution: 15 * minutesMs },
{ maxAge: 96 * hoursMs, resolution: 30 * minutesMs },
];
export const STATS_LOG_SIZE_LIMIT = 720; //144+48+144 (max data snaps) + 384 (1 reboot every 30 mins)
export const PERF_DATA_THREAD_NAMES = ['svNetwork', 'svSync', 'svMain'] as const;
export type PerfDataThreadNamesType = ValuesType<typeof PERF_DATA_THREAD_NAMES>;


/**
Expand Down Expand Up @@ -78,6 +57,13 @@ export default class PerformanceCollector {
resetPerfState() {
this.lastPerfCounts = undefined;
this.lastPerfSaved = undefined;
}


/**
* Reset the last perf data except boundaries
*/
resetMemoryState() {
this.lastNodeMemory = undefined;
this.lastFxsMemory = undefined;
}
Expand All @@ -88,6 +74,7 @@ export default class PerformanceCollector {
*/
logServerBoot(bootTime: number) {
this.resetPerfState();
this.resetMemoryState();
//If last log is a boot, remove it as the server didn't really start
// otherwise it would have lived long enough to have stats logged
if (this.statsLog.length && this.statsLog.at(-1)!.type === 'svBoot') {
Expand All @@ -98,6 +85,7 @@ export default class PerformanceCollector {
type: 'svBoot',
bootTime,
});
this.saveStatsHistory();
}


Expand All @@ -106,6 +94,7 @@ export default class PerformanceCollector {
*/
logServerClose(reason: string) {
this.resetPerfState();
this.resetMemoryState();
if (this.statsLog.length) {
if (this.statsLog.at(-1)!.type === 'svClose') {
//If last log is a close, skip saving a new one
Expand All @@ -121,6 +110,7 @@ export default class PerformanceCollector {
type: 'svClose',
reason,
});
this.saveStatsHistory();
}


Expand Down Expand Up @@ -244,16 +234,10 @@ export default class PerformanceCollector {
perf: perfHistToSave,
};
this.statsLog.push(currSnapshot);
await optimizeStatsLog(this.statsLog);
console.verbose.ok(`Collected performance snapshot #${this.statsLog.length}`);

//Save perf series do file
const savePerfData: SSFileType = {
version: STATS_DATA_FILE_VERSION,
lastPerfBoundaries: this.lastPerfBoundaries,
log: this.statsLog,
};
await fsp.writeFile(this.statsDataPath, JSON.stringify(savePerfData));
console.verbose.ok(`Collected performance snapshot #${this.statsLog.length}`);
await this.saveStatsHistory();
}


Expand All @@ -265,15 +249,38 @@ export default class PerformanceCollector {
const rawFileData = await fsp.readFile(this.statsDataPath, 'utf8');
const fileData = JSON.parse(rawFileData);
if (fileData?.version !== STATS_DATA_FILE_VERSION) throw new Error('invalid version');
const statsData = await SSFileSchema.parseAsync(fileData);
const statsData = SSFileSchema.parse(fileData);
this.lastPerfBoundaries = statsData.lastPerfBoundaries;
this.statsLog = statsData.log;
this.resetPerfState();
console.verbose.debug(`Loaded ${this.statsLog.length} performance snapshots from cache`);
optimizeStatsLog(this.statsLog);
await optimizeStatsLog(this.statsLog);
} catch (error) {
if (error instanceof ZodError) {
console.warn(`Failed to load ${STATS_DATA_FILE_NAME} due to invalid data.`);
console.warn('Since this is not a critical file, it will be reset.');
} else {
console.warn(`Failed to load ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`);
console.warn('Since this is not a critical file, it will be reset.');
}
}
}


/**
* Saves the stats database/cache/history
*/
async saveStatsHistory() {
try {
await optimizeStatsLog(this.statsLog);
const savePerfData: SSFileType = {
version: STATS_DATA_FILE_VERSION,
lastPerfBoundaries: this.lastPerfBoundaries,
log: this.statsLog,
};
await fsp.writeFile(this.statsDataPath, JSON.stringify(savePerfData));
} catch (error) {
console.warn(`Failed to load ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`);
console.warn('Since this is not a critical file, it will be reset.');
console.warn(`Failed to save ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`);
}
}

Expand Down
2 changes: 1 addition & 1 deletion core/components/PerformanceCollector/perfParser.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { PERF_DATA_BUCKET_COUNT } from "./index";
import { PERF_DATA_BUCKET_COUNT } from "./statsConfigs";
import { isValidPerfThreadName, type SSPerfBoundariesType, type SSPerfCountsType } from "./perfSchemas";


Expand Down
4 changes: 2 additions & 2 deletions core/components/PerformanceCollector/perfSchemas.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import * as z from 'zod';
import { PERF_DATA_BUCKET_COUNT, PERF_DATA_THREAD_NAMES, PerfDataThreadNamesType } from './index';
import { PERF_DATA_BUCKET_COUNT, PERF_DATA_THREAD_NAMES, PerfDataThreadNamesType } from './statsConfigs';
import { ValuesType } from 'utility-types';


Expand Down Expand Up @@ -74,7 +74,7 @@ export const SSLogSvCloseSchema = z.object({

export const SSFileSchema = z.object({
version: z.literal(1),
lastPerfBoundaries: SSPerfBoundariesSchema,
lastPerfBoundaries: SSPerfBoundariesSchema.optional(),
log: z.array(z.union([SSLogDataSchema, SSLogSvBootSchema, SSLogSvCloseSchema])),
});

Expand Down
29 changes: 13 additions & 16 deletions core/components/PerformanceCollector/perfUtils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import type { SSPerfCountsType, SSPerfHistType } from "./perfSchemas";
import got from '@core/extras/got.js';
import { parseRawPerf } from './perfParser';
import { getProcessesData } from '@core/webroutes/diagnostics/diagnosticsFuncs';
import { PERF_DATA_BUCKET_COUNT, PerfDataThreadNamesType } from './index';
import { PERF_DATA_BUCKET_COUNT, PerfDataThreadNamesType } from './statsConfigs';


//Consts
Expand All @@ -30,24 +30,22 @@ const perfDataRawThreadsTemplate: SSPerfCountsType = {
* Compares a perf snapshot with the one that came before
*/
export const diffPerfs = (newPerf: SSPerfCountsType, oldPerf?: SSPerfCountsType) => {
if (!oldPerf) {
oldPerf = cloneDeep(perfDataRawThreadsTemplate);
}
const basePerf = oldPerf ?? cloneDeep(perfDataRawThreadsTemplate);
return {
svSync: {
count: newPerf.svSync.count - oldPerf.svSync.count,
// sum: newPerf.svSync.sum - oldPerf.svSync.sum,
buckets: newPerf.svSync.buckets.map((bucket, i) => bucket - oldPerf.svSync.buckets[i]),
count: newPerf.svSync.count - basePerf.svSync.count,
// sum: newPerf.svSync.sum - basePerf.svSync.sum,
buckets: newPerf.svSync.buckets.map((bucket, i) => bucket - basePerf.svSync.buckets[i]),
},
svNetwork: {
count: newPerf.svNetwork.count - oldPerf.svNetwork.count,
// sum: newPerf.svNetwork.sum - oldPerf.svNetwork.sum,
buckets: newPerf.svNetwork.buckets.map((bucket, i) => bucket - oldPerf.svNetwork.buckets[i]),
count: newPerf.svNetwork.count - basePerf.svNetwork.count,
// sum: newPerf.svNetwork.sum - basePerf.svNetwork.sum,
buckets: newPerf.svNetwork.buckets.map((bucket, i) => bucket - basePerf.svNetwork.buckets[i]),
},
svMain: {
count: newPerf.svMain.count - oldPerf.svMain.count,
// sum: newPerf.svMain.sum - oldPerf.svMain.sum,
buckets: newPerf.svMain.buckets.map((bucket, i) => bucket - oldPerf.svMain.buckets[i]),
count: newPerf.svMain.count - basePerf.svMain.count,
// sum: newPerf.svMain.sum - basePerf.svMain.sum,
buckets: newPerf.svMain.buckets.map((bucket, i) => bucket - basePerf.svMain.buckets[i]),
},
};
};
Expand Down Expand Up @@ -78,7 +76,6 @@ export const perfCountsToHist = (threads: SSPerfCountsType) => {
return (bucketValue - prevBucketValue) / tData.count;
});
}

return currPerfFreqs;
}

Expand All @@ -98,9 +95,9 @@ export const fetchRawPerfData = async (fxServerHost: string) => {
export const fetchFxsMemory = async () => {
const allProcsData = await getProcessesData();
if (!allProcsData) return;

const fxProcData = allProcsData.find((proc) => proc.name === 'FXServer');
if (!fxProcData) return;

return fxProcData.memory;
return parseFloat((fxProcData.memory).toFixed(2));
}
22 changes: 22 additions & 0 deletions core/components/PerformanceCollector/statsConfigs.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
import { ValuesType } from "utility-types";


/**
* Configs
*/
const minutesMs = 60 * 1000;
const hoursMs = 60 * minutesMs;
export const PERF_DATA_BUCKET_COUNT = 15;
export const PERF_DATA_MIN_TICKS = 1000; //less than that and the data is not reliable
export const PERF_DATA_INITIAL_RESOLUTION = 5 * minutesMs;
export const STATS_RESOLUTION_TABLE = [
//00~12h = 5min = 12/h = 144 snaps
//12~24h = 15min = 4/h = 48 snaps
//24~96h = 30min = 2/h = 144 snaps
{ maxAge: 12 * hoursMs, resolution: PERF_DATA_INITIAL_RESOLUTION },
{ maxAge: 24 * hoursMs, resolution: 15 * minutesMs },
{ maxAge: 96 * hoursMs, resolution: 30 * minutesMs },
];
export const STATS_LOG_SIZE_LIMIT = 720; //144+48+144 (max data snaps) + 384 (1 reboot every 30 mins)
export const PERF_DATA_THREAD_NAMES = ['svNetwork', 'svSync', 'svMain'] as const;
export type PerfDataThreadNamesType = ValuesType<typeof PERF_DATA_THREAD_NAMES>;
2 changes: 1 addition & 1 deletion core/components/PerformanceCollector/statsLogOptimizer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { STATS_LOG_SIZE_LIMIT, STATS_RESOLUTION_TABLE } from "./index";
import { STATS_LOG_SIZE_LIMIT, STATS_RESOLUTION_TABLE } from "./statsConfigs";
import type { SSLogType } from "./perfSchemas";

//Consts
Expand Down

0 comments on commit 16b21bf

Please sign in to comment.