wip(perf): fetching fxsMemory and perfData in parallel

core/components/PerformanceCollector/index.ts

@@ -1,15 +1,13 @@
 const modulename = 'PerformanceCollector';
 import fsp from 'node:fs/promises';
 import * as d3array from 'd3-array';
-import { convars } from '@core/globalData';
-import got from '@core/extras/got.js';
 import consoleFactory from '@extras/console';
 import type TxAdmin from '@core/txAdmin.js';
 import { PERF_DATA_BUCKET_COUNT, PERF_DATA_MIN_TICKS, SSFileSchema, isSSLogDataType } from './perfSchemas';
 import type { SSFileType, SSLogDataType, SSLogType, SSPerfBucketBoundariesType, SSRawPerfType } from './perfSchemas';
-import { diffPerfs, rawPerfToFreqs } from './perfUtils';
-import { parsePerf } from './perfParser';
+import { diffPerfs, fetchFxsMemory, fetchPerfData, rawPerfToFreqs } from './perfUtils';
 import { optimizeStatsLog } from './statsLogOptimizer';
+import { convars } from '@core/globalData';
 const console = consoleFactory(modulename);
 
 
@@ -21,10 +19,9 @@ const STATS_DATA_FILE_NAME = 'statsData.json';
 export default class PerformanceCollector {
     readonly #txAdmin: TxAdmin;
     private readonly statsDataPath: string;
+    private statsLog: SSLogType = [];
     private lastPerfBucketBoundaries: SSPerfBucketBoundariesType | undefined;
     private lastRawPerf: SSRawPerfType | undefined;
-    private statsLog: SSLogType = [];
-
     private lastNodeMemory: { used: number, total: number } | undefined;
     private lastFxsMemory: number | undefined;
 
@@ -92,25 +89,30 @@ export default class PerformanceCollector {
         if (this.#txAdmin.playerlistManager === null) return;
         if (this.#txAdmin.healthMonitor.currentStatus !== 'ONLINE') return;
 
-        //TODO: Update fxserver memory usage
-        this.lastFxsMemory = undefined;
-        //FIXME: usar promises.all pra paralelizar puxar a performance e a memória
-
         //Get performance data
         const fxServerHost = (convars.debugExternalStatsSource)
             ? convars.debugExternalStatsSource
             : this.#txAdmin.fxRunner.fxServerHost;
-        const currPerfRaw = await got(`http://${fxServerHost}/perf/`).text();
-        const {
-            bucketBoundaries,
-            metrics: currPerfData,
-        } = parsePerf(currPerfRaw);
+        if (typeof fxServerHost !== 'string' || !fxServerHost) {
+            throw new Error(`Invalid fxServerHost: ${fxServerHost}`);
+        }
+
+        //Fetch data
+        const [fetchPerfDataRes, fetchFxsMemoryRes] = await Promise.allSettled([
+            fetchPerfData(fxServerHost),
+            fetchFxsMemory(),
+        ]);
+        if (fetchFxsMemoryRes.status === 'fulfilled') {
+            this.lastFxsMemory = fetchFxsMemoryRes.value;
+        }
+        if (fetchPerfDataRes.status === 'rejected') throw fetchPerfDataRes.reason;
+        const { bucketBoundaries, perfMetrics } = fetchPerfDataRes.value;
 
         //Check for min tick count
         if (
-            currPerfData.svMain.count < PERF_DATA_MIN_TICKS ||
-            currPerfData.svNetwork.count < PERF_DATA_MIN_TICKS ||
-            currPerfData.svSync.count < PERF_DATA_MIN_TICKS
+            perfMetrics.svMain.count < PERF_DATA_MIN_TICKS ||
+            perfMetrics.svNetwork.count < PERF_DATA_MIN_TICKS ||
+            perfMetrics.svSync.count < PERF_DATA_MIN_TICKS
         ) {
             console.verbose.warn('Not enough ticks to log. Skipping this collection.');
             return;
@@ -125,18 +127,18 @@ export default class PerformanceCollector {
             this.statsLog = [];
             this.lastPerfBucketBoundaries = bucketBoundaries;
             this.lastRawPerf = undefined;
-        } else if (this.lastRawPerf.svMain.count > currPerfData.svMain.count) {
+        } else if (this.lastRawPerf.svMain.count > perfMetrics.svMain.count) {
             console.warn('Performance counter reset. Resetting lastRawPerf.');
             this.lastRawPerf = undefined;
         }
 
         //Calculate the tick/time counts since last epoch
-        const currRawPerfDiff = diffPerfs(currPerfData, this.lastRawPerf);
+        const currRawPerfDiff = diffPerfs(perfMetrics, this.lastRawPerf);
         const currPerfLogData = rawPerfToFreqs(currRawPerfDiff);
 
         //Update cache
         const now = Date.now();
-        this.lastRawPerf = currPerfData;
+        this.lastRawPerf = perfMetrics;
         const currSnapshot: SSLogDataType = {
             ts: now,
             type: 'data',

core/components/PerformanceCollector/perfParser.test.ts

@@ -70,11 +70,11 @@ suite('parsePerf', () => {
     it('should parse the perf data correctly', () => {
         const result = parsePerf(perfValidExample);
         expect(result.bucketBoundaries).toEqual([0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, '+Inf']);
-        expect(result.metrics.svNetwork.count).toBe(1840805);
-        expect(result.metrics.svSync.count).toBe(2268704);
-        expect(result.metrics.svMain.count).toBe(355594);
-        expect(result.metrics.svSync.sum).toBe(1091.617999988212);
-        expect(result.metrics.svMain.buckets).toEqual([299261, 327819, 352052, 354360, 354808, 355262, 355577, 355591, 355591, 355592, 355593, 355593, 355593, 355593, 355594]);
+        expect(result.perfMetrics.svNetwork.count).toBe(1840805);
+        expect(result.perfMetrics.svSync.count).toBe(2268704);
+        expect(result.perfMetrics.svMain.count).toBe(355594);
+        // expect(result.perfMetrics.svSync.sum).toBe(1091.617999988212);
+        expect(result.perfMetrics.svMain.buckets).toEqual([299261, 327819, 352052, 354360, 354808, 355262, 355577, 355591, 355591, 355592, 355593, 355593, 355593, 355593, 355594]);
     });
 
     it('should handle bad data', () => {
@@ -88,8 +88,8 @@ suite('parsePerf', () => {
         perfModifiedExample = perfValidExample.replace(targetLine, '');
         expect(() => parsePerf(perfModifiedExample)).toThrow('invalid threads');
 
-        targetLine = 'tickTime_sum{name="svNetwork"} 76.39499999999963';
-        perfModifiedExample = perfValidExample.replace(targetLine, 'tickTime_sum{name="svNetwork"} ????');
+        targetLine = 'tickTime_count{name="svNetwork"} 1840805';
+        perfModifiedExample = perfValidExample.replace(targetLine, 'tickTime_count{name="svNetwork"} ????');
         expect(() => parsePerf(perfModifiedExample)).toThrow('invalid threads');
     });
 });

core/components/PerformanceCollector/perfParser.ts

@@ -1,20 +1,10 @@
-import { PERF_DATA_BUCKET_COUNT, isValidPerfThreadName, type SSPerfBucketBoundariesType, type PerfDataThreadNamesType } from "./perfSchemas";
+import { PERF_DATA_BUCKET_COUNT, isValidPerfThreadName, type SSPerfBucketBoundariesType, type SSRawPerfType } from "./perfSchemas";
 
 
 //Consts
 const REGEX_BUCKET_BOUNDARIE = /le="(\d+(\.\d+)?|\+Inf)"/;
 const REGEX_PERF_LINE = /tickTime_(count|sum|bucket)\{name="(svSync|svNetwork|svMain)"(,le="(\d+(\.\d+)?|\+Inf)")?\}\s(\S+)/;
 
-//Types
-type ParsedRawPerfType = {
-    bucketBoundaries: SSPerfBucketBoundariesType,
-    metrics: Record<PerfDataThreadNamesType, {
-        count: number,
-        sum: number,
-        buckets: number[]
-    }>
-};
-
 
 /**
  * Returns if the given thread name is a valid PerfDataThreadNamesType
@@ -49,23 +39,23 @@ export const arePerfBucketBoundariesValid = (boundaries: (number | string)[]): b
 /**
  * Parses the output of FXServer /perf/ in the proteus format
  */
-export const parsePerf = (rawData: string): ParsedRawPerfType => {
+export const parsePerf = (rawData: string) => {
     if (typeof rawData !== 'string') throw new Error('string expected');
     const lines = rawData.trim().split('\n');
-    const metrics: ParsedRawPerfType['metrics'] = {
+    const perfMetrics: SSRawPerfType = {
         svSync: {
             count: 0,
-            sum: 0,
+            // sum: 0,
             buckets: [],
         },
         svNetwork: {
             count: 0,
-            sum: 0,
+            // sum: 0,
             buckets: [],
         },
         svMain: {
             count: 0,
-            sum: 0,
+            // sum: 0,
             buckets: [],
         },
     };
@@ -102,13 +92,13 @@ export const parsePerf = (rawData: string): ParsedRawPerfType => {
 
         if (regType == 'count') {
             const count = parseInt(value);
-            if (!isNaN(count)) metrics[thread].count = count;
+            if (!isNaN(count)) perfMetrics[thread].count = count;
         } else if (regType == 'sum') {
-            const sum = parseFloat(value);
-            if (!isNaN(sum)) metrics[thread].sum = sum;
+            // const sum = parseFloat(value);
+            // if (!isNaN(sum)) currPerfData[thread].sum = sum;
         } else if (regType == 'bucket') {
             //Check if the bucket is correct
-            const currBucketIndex = metrics[thread].buckets.length;
+            const currBucketIndex = perfMetrics[thread].buckets.length;
             const lastBucketIndex = PERF_DATA_BUCKET_COUNT - 1;
             if (currBucketIndex === lastBucketIndex) {
                 if (bucket !== '+Inf') {
@@ -118,23 +108,23 @@ export const parsePerf = (rawData: string): ParsedRawPerfType => {
                 throw new Error(`unexpected bucket ${bucket} at position ${currBucketIndex}`);
             }
             //Add the bucket
-            metrics[thread].buckets.push(parseInt(value));
+            perfMetrics[thread].buckets.push(parseInt(value));
         }
     }
 
     //Check perf validity
-    const invalid = Object.values(metrics).filter((thread) => {
+    const invalid = Object.values(perfMetrics).filter((thread) => {
         return (
             !Number.isInteger(thread.count)
             || thread.count === 0
-            || !Number.isFinite(thread.sum)
-            || thread.sum === 0
+            // || !Number.isFinite(thread.sum)
+            // || thread.sum === 0
             || thread.buckets.length !== PERF_DATA_BUCKET_COUNT
         );
     });
     if (invalid.length) {
         throw new Error(`${invalid.length} invalid threads in /perf/`);
     }
 
-    return { bucketBoundaries, metrics };
+    return { bucketBoundaries, perfMetrics };
 };

core/components/PerformanceCollector/perfUtils.ts

@@ -1,6 +1,9 @@
 import { cloneDeep } from 'lodash-es';
 import { PERF_DATA_BUCKET_COUNT } from "./perfSchemas";
-import type {SSRawPerfType, SSLogDataPerfType, PerfDataThreadNamesType} from "./perfSchemas";
+import type { SSRawPerfType, SSLogDataPerfType, PerfDataThreadNamesType } from "./perfSchemas";
+import got from '@core/extras/got.js';
+import { parsePerf } from './perfParser';
+import { getProcessesData } from '@core/webroutes/diagnostics/diagnosticsFuncs';
 
 
 //Consts
@@ -78,3 +81,26 @@ export const rawPerfToFreqs = (threads: SSRawPerfType) => {
 
     return currPerfFreqs;
 }
+
+
+/**
+ * Requests /perf/, parses it and returns the raw perf data
+ */
+export const fetchPerfData = async (fxServerHost: string) => {
+    const currPerfRaw = await got(`http://${fxServerHost}/perf/`).text();
+    return parsePerf(currPerfRaw);
+}
+
+
+/**
+ * Get the fxserver memory usage
+ */
+export const fetchFxsMemory = async () => {
+    const allProcsData = await getProcessesData();
+    if (!allProcsData) return;
+
+    const fxProcData = allProcsData.find((proc) => proc.name === 'FXServer');
+    if (!fxProcData) return;
+
+    return fxProcData.memory;
+}