diff --git a/core/components/PerformanceCollector/index.ts b/core/components/PerformanceCollector/index.ts index c3df6d753..90b270870 100644 --- a/core/components/PerformanceCollector/index.ts +++ b/core/components/PerformanceCollector/index.ts @@ -8,36 +8,15 @@ import type { LogNodeHeapEventType, SSFileType, SSLogDataType, SSLogType, SSPerf import { diffPerfs, fetchFxsMemory, fetchRawPerfData, perfCountsToHist } from './perfUtils'; import { optimizeStatsLog } from './statsLogOptimizer'; import { convars } from '@core/globalData'; -import { ValuesType } from 'utility-types'; -import bytes from 'bytes'; +import { ZodError } from 'zod'; +import { PERF_DATA_BUCKET_COUNT, PERF_DATA_INITIAL_RESOLUTION, PERF_DATA_MIN_TICKS } from './statsConfigs'; const console = consoleFactory(modulename); //Consts -const minutesMs = 60 * 1000; -const hoursMs = 60 * minutesMs; const megabyte = 1024 * 1024; - - -/** - * Configs - */ const STATS_DATA_FILE_VERSION = 1; const STATS_DATA_FILE_NAME = 'statsData.json'; -export const PERF_DATA_BUCKET_COUNT = 15; -export const PERF_DATA_MIN_TICKS = 2000; //less than that and the data is not reliable -export const PERF_DATA_INITIAL_RESOLUTION = 5 * minutesMs; -export const STATS_RESOLUTION_TABLE = [ - //00~12h = 5min = 12/h = 144 snaps - //12~24h = 15min = 4/h = 48 snaps - //24~96h = 30min = 2/h = 144 snaps - { maxAge: 12 * hoursMs, resolution: PERF_DATA_INITIAL_RESOLUTION }, - { maxAge: 24 * hoursMs, resolution: 15 * minutesMs }, - { maxAge: 96 * hoursMs, resolution: 30 * minutesMs }, -]; -export const STATS_LOG_SIZE_LIMIT = 720; //144+48+144 (max data snaps) + 384 (1 reboot every 30 mins) -export const PERF_DATA_THREAD_NAMES = ['svNetwork', 'svSync', 'svMain'] as const; -export type PerfDataThreadNamesType = ValuesType; /** @@ -78,6 +57,13 @@ export default class PerformanceCollector { resetPerfState() { this.lastPerfCounts = undefined; this.lastPerfSaved = undefined; + } + + + /** + * Reset the last perf data except boundaries + */ + resetMemoryState() { this.lastNodeMemory = undefined; this.lastFxsMemory = undefined; } @@ -88,6 +74,7 @@ export default class PerformanceCollector { */ logServerBoot(bootTime: number) { this.resetPerfState(); + this.resetMemoryState(); //If last log is a boot, remove it as the server didn't really start // otherwise it would have lived long enough to have stats logged if (this.statsLog.length && this.statsLog.at(-1)!.type === 'svBoot') { @@ -98,6 +85,7 @@ export default class PerformanceCollector { type: 'svBoot', bootTime, }); + this.saveStatsHistory(); } @@ -106,6 +94,7 @@ export default class PerformanceCollector { */ logServerClose(reason: string) { this.resetPerfState(); + this.resetMemoryState(); if (this.statsLog.length) { if (this.statsLog.at(-1)!.type === 'svClose') { //If last log is a close, skip saving a new one @@ -121,6 +110,7 @@ export default class PerformanceCollector { type: 'svClose', reason, }); + this.saveStatsHistory(); } @@ -244,16 +234,10 @@ export default class PerformanceCollector { perf: perfHistToSave, }; this.statsLog.push(currSnapshot); - await optimizeStatsLog(this.statsLog); + console.verbose.ok(`Collected performance snapshot #${this.statsLog.length}`); //Save perf series do file - const savePerfData: SSFileType = { - version: STATS_DATA_FILE_VERSION, - lastPerfBoundaries: this.lastPerfBoundaries, - log: this.statsLog, - }; - await fsp.writeFile(this.statsDataPath, JSON.stringify(savePerfData)); - console.verbose.ok(`Collected performance snapshot #${this.statsLog.length}`); + await this.saveStatsHistory(); } @@ -265,15 +249,38 @@ export default class PerformanceCollector { const rawFileData = await fsp.readFile(this.statsDataPath, 'utf8'); const fileData = JSON.parse(rawFileData); if (fileData?.version !== STATS_DATA_FILE_VERSION) throw new Error('invalid version'); - const statsData = await SSFileSchema.parseAsync(fileData); + const statsData = SSFileSchema.parse(fileData); this.lastPerfBoundaries = statsData.lastPerfBoundaries; this.statsLog = statsData.log; this.resetPerfState(); console.verbose.debug(`Loaded ${this.statsLog.length} performance snapshots from cache`); - optimizeStatsLog(this.statsLog); + await optimizeStatsLog(this.statsLog); + } catch (error) { + if (error instanceof ZodError) { + console.warn(`Failed to load ${STATS_DATA_FILE_NAME} due to invalid data.`); + console.warn('Since this is not a critical file, it will be reset.'); + } else { + console.warn(`Failed to load ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`); + console.warn('Since this is not a critical file, it will be reset.'); + } + } + } + + + /** + * Saves the stats database/cache/history + */ + async saveStatsHistory() { + try { + await optimizeStatsLog(this.statsLog); + const savePerfData: SSFileType = { + version: STATS_DATA_FILE_VERSION, + lastPerfBoundaries: this.lastPerfBoundaries, + log: this.statsLog, + }; + await fsp.writeFile(this.statsDataPath, JSON.stringify(savePerfData)); } catch (error) { - console.warn(`Failed to load ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`); - console.warn('Since this is not a critical file, it will be reset.'); + console.warn(`Failed to save ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`); } } diff --git a/core/components/PerformanceCollector/perfParser.ts b/core/components/PerformanceCollector/perfParser.ts index 681f0ddfe..baf64eff0 100644 --- a/core/components/PerformanceCollector/perfParser.ts +++ b/core/components/PerformanceCollector/perfParser.ts @@ -1,4 +1,4 @@ -import { PERF_DATA_BUCKET_COUNT } from "./index"; +import { PERF_DATA_BUCKET_COUNT } from "./statsConfigs"; import { isValidPerfThreadName, type SSPerfBoundariesType, type SSPerfCountsType } from "./perfSchemas"; diff --git a/core/components/PerformanceCollector/perfSchemas.ts b/core/components/PerformanceCollector/perfSchemas.ts index 32e2fc4a4..7521bbb73 100644 --- a/core/components/PerformanceCollector/perfSchemas.ts +++ b/core/components/PerformanceCollector/perfSchemas.ts @@ -1,5 +1,5 @@ import * as z from 'zod'; -import { PERF_DATA_BUCKET_COUNT, PERF_DATA_THREAD_NAMES, PerfDataThreadNamesType } from './index'; +import { PERF_DATA_BUCKET_COUNT, PERF_DATA_THREAD_NAMES, PerfDataThreadNamesType } from './statsConfigs'; import { ValuesType } from 'utility-types'; @@ -74,7 +74,7 @@ export const SSLogSvCloseSchema = z.object({ export const SSFileSchema = z.object({ version: z.literal(1), - lastPerfBoundaries: SSPerfBoundariesSchema, + lastPerfBoundaries: SSPerfBoundariesSchema.optional(), log: z.array(z.union([SSLogDataSchema, SSLogSvBootSchema, SSLogSvCloseSchema])), }); diff --git a/core/components/PerformanceCollector/perfUtils.ts b/core/components/PerformanceCollector/perfUtils.ts index e06f7ffea..aa6997dd7 100644 --- a/core/components/PerformanceCollector/perfUtils.ts +++ b/core/components/PerformanceCollector/perfUtils.ts @@ -3,7 +3,7 @@ import type { SSPerfCountsType, SSPerfHistType } from "./perfSchemas"; import got from '@core/extras/got.js'; import { parseRawPerf } from './perfParser'; import { getProcessesData } from '@core/webroutes/diagnostics/diagnosticsFuncs'; -import { PERF_DATA_BUCKET_COUNT, PerfDataThreadNamesType } from './index'; +import { PERF_DATA_BUCKET_COUNT, PerfDataThreadNamesType } from './statsConfigs'; //Consts @@ -30,24 +30,22 @@ const perfDataRawThreadsTemplate: SSPerfCountsType = { * Compares a perf snapshot with the one that came before */ export const diffPerfs = (newPerf: SSPerfCountsType, oldPerf?: SSPerfCountsType) => { - if (!oldPerf) { - oldPerf = cloneDeep(perfDataRawThreadsTemplate); - } + const basePerf = oldPerf ?? cloneDeep(perfDataRawThreadsTemplate); return { svSync: { - count: newPerf.svSync.count - oldPerf.svSync.count, - // sum: newPerf.svSync.sum - oldPerf.svSync.sum, - buckets: newPerf.svSync.buckets.map((bucket, i) => bucket - oldPerf.svSync.buckets[i]), + count: newPerf.svSync.count - basePerf.svSync.count, + // sum: newPerf.svSync.sum - basePerf.svSync.sum, + buckets: newPerf.svSync.buckets.map((bucket, i) => bucket - basePerf.svSync.buckets[i]), }, svNetwork: { - count: newPerf.svNetwork.count - oldPerf.svNetwork.count, - // sum: newPerf.svNetwork.sum - oldPerf.svNetwork.sum, - buckets: newPerf.svNetwork.buckets.map((bucket, i) => bucket - oldPerf.svNetwork.buckets[i]), + count: newPerf.svNetwork.count - basePerf.svNetwork.count, + // sum: newPerf.svNetwork.sum - basePerf.svNetwork.sum, + buckets: newPerf.svNetwork.buckets.map((bucket, i) => bucket - basePerf.svNetwork.buckets[i]), }, svMain: { - count: newPerf.svMain.count - oldPerf.svMain.count, - // sum: newPerf.svMain.sum - oldPerf.svMain.sum, - buckets: newPerf.svMain.buckets.map((bucket, i) => bucket - oldPerf.svMain.buckets[i]), + count: newPerf.svMain.count - basePerf.svMain.count, + // sum: newPerf.svMain.sum - basePerf.svMain.sum, + buckets: newPerf.svMain.buckets.map((bucket, i) => bucket - basePerf.svMain.buckets[i]), }, }; }; @@ -78,7 +76,6 @@ export const perfCountsToHist = (threads: SSPerfCountsType) => { return (bucketValue - prevBucketValue) / tData.count; }); } - return currPerfFreqs; } @@ -98,9 +95,9 @@ export const fetchRawPerfData = async (fxServerHost: string) => { export const fetchFxsMemory = async () => { const allProcsData = await getProcessesData(); if (!allProcsData) return; - + const fxProcData = allProcsData.find((proc) => proc.name === 'FXServer'); if (!fxProcData) return; - return fxProcData.memory; + return parseFloat((fxProcData.memory).toFixed(2)); } diff --git a/core/components/PerformanceCollector/statsConfigs.ts b/core/components/PerformanceCollector/statsConfigs.ts new file mode 100644 index 000000000..9ecd60774 --- /dev/null +++ b/core/components/PerformanceCollector/statsConfigs.ts @@ -0,0 +1,22 @@ +import { ValuesType } from "utility-types"; + + +/** + * Configs + */ +const minutesMs = 60 * 1000; +const hoursMs = 60 * minutesMs; +export const PERF_DATA_BUCKET_COUNT = 15; +export const PERF_DATA_MIN_TICKS = 1000; //less than that and the data is not reliable +export const PERF_DATA_INITIAL_RESOLUTION = 5 * minutesMs; +export const STATS_RESOLUTION_TABLE = [ + //00~12h = 5min = 12/h = 144 snaps + //12~24h = 15min = 4/h = 48 snaps + //24~96h = 30min = 2/h = 144 snaps + { maxAge: 12 * hoursMs, resolution: PERF_DATA_INITIAL_RESOLUTION }, + { maxAge: 24 * hoursMs, resolution: 15 * minutesMs }, + { maxAge: 96 * hoursMs, resolution: 30 * minutesMs }, +]; +export const STATS_LOG_SIZE_LIMIT = 720; //144+48+144 (max data snaps) + 384 (1 reboot every 30 mins) +export const PERF_DATA_THREAD_NAMES = ['svNetwork', 'svSync', 'svMain'] as const; +export type PerfDataThreadNamesType = ValuesType; diff --git a/core/components/PerformanceCollector/statsLogOptimizer.ts b/core/components/PerformanceCollector/statsLogOptimizer.ts index 479630334..7c3620074 100644 --- a/core/components/PerformanceCollector/statsLogOptimizer.ts +++ b/core/components/PerformanceCollector/statsLogOptimizer.ts @@ -1,4 +1,4 @@ -import { STATS_LOG_SIZE_LIMIT, STATS_RESOLUTION_TABLE } from "./index"; +import { STATS_LOG_SIZE_LIMIT, STATS_RESOLUTION_TABLE } from "./statsConfigs"; import type { SSLogType } from "./perfSchemas"; //Consts