diff --git a/src/lib/GraphDataUtils.js b/src/lib/GraphDataUtils.js index 39a694304..57a2bc252 100644 --- a/src/lib/GraphDataUtils.js +++ b/src/lib/GraphDataUtils.js @@ -490,6 +490,11 @@ export function processPieData(data, valueColumn, groupByColumn, aggregationType if (groupByColumn) { // Group calculated values by the same groupByColumn const groups = {}; + // For percent operator, track numerator/denominator separately + const percentComponents = {}; + // For average operator, track field values separately + const averageComponents = {}; + rowsWithCalcValues.forEach(({ item, calculatedValues: calcVals }) => { const calcValue = calcVals[calc.name]; if (calcValue !== null) { @@ -498,6 +503,39 @@ export function processPieData(data, valueColumn, groupByColumn, aggregationType groups[groupKey] = []; } groups[groupKey].push(calcValue); + + const enhancedItem = { ...item }; + if (item.attributes) { + enhancedItem.attributes = { ...item.attributes, ...calcVals }; + } else { + Object.assign(enhancedItem, calcVals); + } + + // For percent operator, also track raw numerator/denominator + if (calc.operator === 'percent' && calc.fields && calc.fields.length >= 2) { + const numVal = extractNumericValue(getNestedValue(enhancedItem, calc.fields[0])); + const denVal = extractNumericValue(getNestedValue(enhancedItem, calc.fields[1])); + if (numVal !== null && denVal !== null) { + if (!percentComponents[groupKey]) { + percentComponents[groupKey] = { numerators: [], denominators: [] }; + } + percentComponents[groupKey].numerators.push(numVal); + percentComponents[groupKey].denominators.push(denVal); + } + } + + // For average operator, track individual field values + if (calc.operator === 'average' && calc.fields && calc.fields.length > 0) { + if (!averageComponents[groupKey]) { + averageComponents[groupKey] = { values: [], numFields: calc.fields.length }; + } + calc.fields.forEach(field => { + const numVal = extractNumericValue(getNestedValue(enhancedItem, field)); + if (numVal !== null) { + averageComponents[groupKey].values.push(numVal); + } + }); + } } }); @@ -506,15 +544,27 @@ export function processPieData(data, valueColumn, groupByColumn, aggregationType const labelKey = valueColumns.length > 1 || calculatedValues.length > 1 ? `${calc.name} (${groupKey})` : groupKey; - // For ratio-based operators (percent, ratio, formula), average the results - // For other operators, sum the results - let aggType = 'sum'; - if (calc.operator === 'percent' || calc.operator === 'ratio' || calc.operator === 'formula') { - aggType = 'avg'; - } else if (calc.operator === 'average') { - aggType = 'avg'; + + // For percent operator, calculate (sum of numerators / sum of denominators) * 100 + if (calc.operator === 'percent' && percentComponents[groupKey]) { + const components = percentComponents[groupKey]; + const sumNumerator = components.numerators.reduce((acc, val) => acc + val, 0); + const sumDenominator = components.denominators.reduce((acc, val) => acc + val, 0); + aggregatedData[labelKey] = sumDenominator !== 0 ? (sumNumerator / sumDenominator) * 100 : 0; + } else if (calc.operator === 'average' && averageComponents[groupKey]) { + // For average operator, calculate (sum of all field values) / numFields + const components = averageComponents[groupKey]; + const sumValues = components.values.reduce((acc, val) => acc + val, 0); + aggregatedData[labelKey] = components.numFields > 0 ? sumValues / components.numFields : 0; + } else { + // For other ratio-based operators (ratio, formula), average the results + // For other operators, sum the results + let aggType = 'sum'; + if (calc.operator === 'ratio' || calc.operator === 'formula') { + aggType = 'avg'; + } + aggregatedData[labelKey] = aggregateValues(groups[groupKey], aggType); } - aggregatedData[labelKey] = aggregateValues(groups[groupKey], aggType); }); } else { // No grouping - aggregate all calculated values together @@ -523,15 +573,53 @@ export function processPieData(data, valueColumn, groupByColumn, aggregationType .filter(val => val !== null); if (calcValues.length > 0) { - // For ratio-based operators (percent, ratio, formula), average the results - // For other operators, sum the results - let aggType = 'sum'; - if (calc.operator === 'percent' || calc.operator === 'ratio' || calc.operator === 'formula') { - aggType = 'avg'; - } else if (calc.operator === 'average') { - aggType = 'avg'; + // For percent operator, calculate (sum of numerators / sum of denominators) * 100 + if (calc.operator === 'percent' && calc.fields && calc.fields.length >= 2) { + let sumNumerator = 0; + let sumDenominator = 0; + rowsWithCalcValues.forEach(({ item, calculatedValues: calcVals }) => { + const enhancedItem = { ...item }; + if (item.attributes) { + enhancedItem.attributes = { ...item.attributes, ...calcVals }; + } else { + Object.assign(enhancedItem, calcVals); + } + const numVal = extractNumericValue(getNestedValue(enhancedItem, calc.fields[0])); + const denVal = extractNumericValue(getNestedValue(enhancedItem, calc.fields[1])); + if (numVal !== null && denVal !== null) { + sumNumerator += numVal; + sumDenominator += denVal; + } + }); + aggregatedData[calc.name] = sumDenominator !== 0 ? (sumNumerator / sumDenominator) * 100 : 0; + } else if (calc.operator === 'average' && calc.fields && calc.fields.length > 0) { + // For average operator, calculate (sum of all field values) / numFields + let sumValues = 0; + const numFields = calc.fields.length; + rowsWithCalcValues.forEach(({ item, calculatedValues: calcVals }) => { + const enhancedItem = { ...item }; + if (item.attributes) { + enhancedItem.attributes = { ...item.attributes, ...calcVals }; + } else { + Object.assign(enhancedItem, calcVals); + } + calc.fields.forEach(field => { + const numVal = extractNumericValue(getNestedValue(enhancedItem, field)); + if (numVal !== null) { + sumValues += numVal; + } + }); + }); + aggregatedData[calc.name] = numFields > 0 ? sumValues / numFields : 0; + } else { + // For other ratio-based operators (ratio, formula), average the results + // For other operators, sum the results + let aggType = 'sum'; + if (calc.operator === 'ratio' || calc.operator === 'formula') { + aggType = 'avg'; + } + aggregatedData[calc.name] = aggregateValues(calcValues, aggType); } - aggregatedData[calc.name] = aggregateValues(calcValues, aggType); } } } @@ -585,6 +673,12 @@ export function processBarLineData(data, xColumn, valueColumn, groupByColumn, ag // Collect unique x-axis values and group data const xValues = new Map(); // Use Map to store both raw value and formatted label const groups = {}; + // Special tracking for percent operator - stores raw numerator/denominator values + // so we can calculate (sum of numerators / sum of denominators) * 100 instead of averaging percentages + const percentComponents = {}; + // Special tracking for average operator - stores individual field values + // so we can calculate (sum of all field values) / numFields instead of averaging per-row averages + const averageComponents = {}; let isDateAxis = false; let hasNonDateAxisValue = false; @@ -690,6 +784,49 @@ export function processBarLineData(data, xColumn, valueColumn, groupByColumn, ag if (!groups[groupKey][xKey]) { groups[groupKey][xKey] = []; } + + // For percent operator, store raw numerator/denominator values separately + // so we can calculate (sum of numerators / sum of denominators) * 100 + // instead of averaging individual percentages + if (calc.operator === 'percent' && calc.fields && calc.fields.length >= 2) { + // Extract numerator and denominator values + const numeratorValue = getNestedValue(enhancedItem, calc.fields[0]); + const denominatorValue = getNestedValue(enhancedItem, calc.fields[1]); + const numVal = extractNumericValue(numeratorValue); + const denVal = extractNumericValue(denominatorValue); + + if (numVal !== null && denVal !== null) { + if (!percentComponents[groupKey]) { + percentComponents[groupKey] = {}; + } + if (!percentComponents[groupKey][xKey]) { + percentComponents[groupKey][xKey] = { numerators: [], denominators: [] }; + } + percentComponents[groupKey][xKey].numerators.push(numVal); + percentComponents[groupKey][xKey].denominators.push(denVal); + } + } + + // For average operator, store individual field values separately + // so we can calculate (sum of all field values) / numFields + // instead of averaging individual per-row averages + if (calc.operator === 'average' && calc.fields && calc.fields.length > 0) { + if (!averageComponents[groupKey]) { + averageComponents[groupKey] = {}; + } + if (!averageComponents[groupKey][xKey]) { + averageComponents[groupKey][xKey] = { values: [], numFields: calc.fields.length }; + } + // Store each field value + calc.fields.forEach(field => { + const fieldValue = getNestedValue(enhancedItem, field); + const numVal = extractNumericValue(fieldValue); + if (numVal !== null) { + averageComponents[groupKey][xKey].values.push(numVal); + } + }); + } + groups[groupKey][xKey].push(calcValue); } } @@ -762,13 +899,27 @@ export function processBarLineData(data, xColumn, valueColumn, groupByColumn, ag const calcOperator = calcValueOperatorMap.get(groupKey); if (calcOperator) { - // For calculated values, the operator has already been applied at row level - // For ratio-based operators (percent, ratio, formula), we should average the results - // For other operators (sum, average, difference), we sum the results + // Special handling for percent operator: calculate (sum of numerators / sum of denominators) * 100 + // This gives the correct percentage of totals rather than average of individual percentages + if (calcOperator === 'percent' && percentComponents[groupKey] && percentComponents[groupKey][xKey]) { + const components = percentComponents[groupKey][xKey]; + const sumNumerator = components.numerators.reduce((acc, val) => acc + val, 0); + const sumDenominator = components.denominators.reduce((acc, val) => acc + val, 0); + return sumDenominator !== 0 ? (sumNumerator / sumDenominator) * 100 : 0; + } + + // Special handling for average operator: calculate (sum of all field values) / numFields + // This gives the correct average of totals rather than average of individual per-row averages + if (calcOperator === 'average' && averageComponents[groupKey] && averageComponents[groupKey][xKey]) { + const components = averageComponents[groupKey][xKey]; + const sumValues = components.values.reduce((acc, val) => acc + val, 0); + return components.numFields > 0 ? sumValues / components.numFields : 0; + } + + // For other ratio-based operators (ratio, formula), average the results + // For other operators (sum, difference), sum the results let aggregationType = 'sum'; - if (calcOperator === 'percent' || calcOperator === 'ratio' || calcOperator === 'formula') { - aggregationType = 'avg'; - } else if (calcOperator === 'average') { + if (calcOperator === 'ratio' || calcOperator === 'formula') { aggregationType = 'avg'; } return groupValues.length > 0 ? aggregateValues(groupValues, aggregationType) : 0; diff --git a/src/lib/tests/GraphDataUtils.test.js b/src/lib/tests/GraphDataUtils.test.js index 12ee9be3e..18669cd83 100644 --- a/src/lib/tests/GraphDataUtils.test.js +++ b/src/lib/tests/GraphDataUtils.test.js @@ -292,5 +292,227 @@ describe('GraphDataUtils', () => { expect(result.labels).toEqual(['A', 'B']); }); }); + + describe('processBarLineData with percent operator', () => { + it('should calculate percent from summed numerators and denominators', () => { + // Test data with multiple rows per month + // Each row has numerator and denominator fields + const mockData = [ + { attributes: { month: 'Jan', numerator: 100, denominator: 10 } }, + { attributes: { month: 'Jan', numerator: 200, denominator: 25 } }, + { attributes: { month: 'Feb', numerator: 150, denominator: 30 } }, + ]; + + const calculatedValues = [{ + name: 'ConversionRate', + operator: 'percent', + fields: ['numerator', 'denominator'], + }]; + + const result = processBarLineData(mockData, 'month', null, null, 'sum', calculatedValues); + + expect(result).toHaveProperty('datasets'); + expect(result.datasets.length).toBe(1); + expect(result.datasets[0].label).toBe('ConversionRate'); + + // Find values for Jan and Feb + const janIndex = result.labels.indexOf('Jan'); + const febIndex = result.labels.indexOf('Feb'); + + // Jan: (100+200)/(10+25)*100 = 300/35*100 = 857.14% + // This is the percentage of totals, not average of individual percentages + expect(result.datasets[0].data[janIndex]).toBeCloseTo(857.14, 1); + + // Feb: 150/30*100 = 500% + expect(result.datasets[0].data[febIndex]).toBe(500); + }); + + it('should handle percent calculation with value columns', () => { + // Test that percent calculated values work correctly alongside regular value columns + const mockData = [ + { attributes: { month: 'Jan', numerator: 443300, denominator: 54008, revenue: 1000 } }, + ]; + + const calculatedValues = [{ + name: 'Percent', + operator: 'percent', + fields: ['numerator', 'denominator'], + }]; + + const result = processBarLineData(mockData, 'month', 'revenue', null, 'sum', calculatedValues); + + expect(result).toHaveProperty('datasets'); + expect(result.datasets.length).toBe(2); // revenue + Percent + + const percentDataset = result.datasets.find(d => d.label === 'Percent'); + expect(percentDataset).toBeDefined(); + + // 443300 / 54008 * 100 = 820.8043... + expect(percentDataset.data[0]).toBeCloseTo(820.8, 1); + }); + + it('should calculate percent using another calculated value as input', () => { + // Test scenario: user has calculated values where one uses another as input + // CalcValue1: Sum of fieldA (e.g., 443300 total) + // CalcValue2: Sum of fieldB (e.g., 54008 total) + // PercentCalc: Percent with numerator=CalcValue1, denominator=CalcValue2 + const mockData = [ + { attributes: { month: 'Jan', fieldA: 100000, fieldB: 12000 } }, + { attributes: { month: 'Jan', fieldA: 200000, fieldB: 22000 } }, + { attributes: { month: 'Jan', fieldA: 143300, fieldB: 20008 } }, + ]; + // Total fieldA: 443300, Total fieldB: 54008 + + const calculatedValues = [ + { + name: 'TotalA', + operator: 'sum', + fields: ['fieldA'], + }, + { + name: 'TotalB', + operator: 'sum', + fields: ['fieldB'], + }, + { + name: 'PercentOfTotals', + operator: 'percent', + fields: ['TotalA', 'TotalB'], + }, + ]; + + const result = processBarLineData(mockData, 'month', null, null, 'sum', calculatedValues); + + expect(result).toHaveProperty('datasets'); + + const percentDataset = result.datasets.find(d => d.label === 'PercentOfTotals'); + expect(percentDataset).toBeDefined(); + + // With the fix, percent is calculated as (sum of numerators / sum of denominators) * 100 + // TotalA per row: 100000, 200000, 143300 -> these are the numerator values + // TotalB per row: 12000, 22000, 20008 -> these are the denominator values + // Percent = (100000+200000+143300) / (12000+22000+20008) * 100 + // = 443300 / 54008 * 100 = 820.8% + expect(percentDataset.data[0]).toBeCloseTo(820.8, 0); + }); + + it('should handle percent with 2 value fields and 2 calculated values (user scenario)', () => { + // User scenario: line chart with x-axis date, 2 value fields (sum), 2 calculated values + const mockData = [ + { attributes: { date: '2024-01-01', numerator: 55412, denominator: 6751, valueA: 100, valueB: 50 } }, + { attributes: { date: '2024-01-01', numerator: 55412, denominator: 6751, valueA: 200, valueB: 60 } }, + { attributes: { date: '2024-01-01', numerator: 55413, denominator: 6751, valueA: 150, valueB: 70 } }, + { attributes: { date: '2024-01-01', numerator: 55413, denominator: 6751, valueA: 120, valueB: 80 } }, + { attributes: { date: '2024-01-01', numerator: 55413, denominator: 6751, valueA: 180, valueB: 90 } }, + { attributes: { date: '2024-01-01', numerator: 55413, denominator: 6751, valueA: 160, valueB: 55 } }, + { attributes: { date: '2024-01-01', numerator: 55412, denominator: 6751, valueA: 140, valueB: 65 } }, + { attributes: { date: '2024-01-01', numerator: 55412, denominator: 6751, valueA: 130, valueB: 75 } }, + ]; + // Total numerator: 443300, Total denominator: 54008 + // Each row: (55412/6751)*100 = 820.8% or (55413/6751)*100 = 820.8% + + const calculatedValues = [ + { + name: 'SomeCalc', + operator: 'sum', + fields: ['valueA', 'valueB'], + }, + { + name: 'PercentCalc', + operator: 'percent', + fields: ['numerator', 'denominator'], + }, + ]; + + // 2 value fields with aggregation 'sum', plus 2 calculated values + const result = processBarLineData(mockData, 'date', ['valueA', 'valueB'], null, 'sum', calculatedValues); + + expect(result).toHaveProperty('datasets'); + + const percentDataset = result.datasets.find(d => d.label === 'PercentCalc'); + expect(percentDataset).toBeDefined(); + + // Expected: average of 8 rows each at ~820.8% = 820.8% (NOT 8 * 820.8 = 6566.4) + expect(percentDataset.data[0]).toBeCloseTo(820.8, 0); + }); + }); + + describe('processBarLineData with average operator', () => { + it('should calculate average from aggregated field sums, not average of per-row averages', () => { + // 3 rows with different values that would give different results + // depending on whether we average per-row averages vs average of sums + const mockData = [ + { date: '2024-01', a: 0, b: 1 }, + { date: '2024-01', a: 0, b: 1 }, + { date: '2024-01', a: 0, b: 1 }, + ]; + + const calculatedValues = [ + { + name: 'avg', + operator: 'average', + fields: ['a', 'b'], + }, + ]; + + const result = processBarLineData(mockData, 'date', ['a', 'b'], null, 'sum', calculatedValues); + + expect(result).toHaveProperty('datasets'); + + // Value columns with sum aggregation + const aDataset = result.datasets.find(d => d.label === 'a'); + const bDataset = result.datasets.find(d => d.label === 'b'); + expect(aDataset.data[0]).toBe(0); // sum of a: 0+0+0 = 0 + expect(bDataset.data[0]).toBe(3); // sum of b: 1+1+1 = 3 + + // Average calculated value + const avgDataset = result.datasets.find(d => d.label === 'avg'); + expect(avgDataset).toBeDefined(); + + // Expected: (sum_a + sum_b) / 2 = (0 + 3) / 2 = 1.5 + // NOT: average of per-row averages = ((0+1)/2 + (0+1)/2 + (0+1)/2) = 0.5+0.5+0.5 = 1.5 summed = wrong + // Wait, per-row avg = 0.5 each, and if we SUM those we get 1.5, which equals (0+3)/2 + // Let me use different values to distinguish + expect(avgDataset.data[0]).toBe(1.5); // (0 + 3) / 2 = 1.5 + }); + + it('should calculate average correctly with asymmetric data', () => { + // Use data where average of per-row averages differs from average of sums + const mockData = [ + { date: '2024-01', a: 2, b: 0 }, // per-row avg = 1 + { date: '2024-01', a: 0, b: 0 }, // per-row avg = 0 + { date: '2024-01', a: 0, b: 1 }, // per-row avg = 0.5 + ]; + // Sum of per-row avgs = 1.5, if summed (wrong aggregation) + // Sum of a = 2, Sum of b = 1 + // Average of sums = (2 + 1) / 2 = 1.5 + // Hmm, still the same! Let me try another approach. + + // Actually the issue is: per-row avg summed vs per-row avg averaged + // Per-row avgs: [1, 0, 0.5] + // If we SUM per-row avgs: 1.5 + // If we AVG per-row avgs: 0.5 + // Average of field sums: (2+1)/2 = 1.5 + + const calculatedValues = [ + { + name: 'avg', + operator: 'average', + fields: ['a', 'b'], + }, + ]; + + const result = processBarLineData(mockData, 'date', ['a', 'b'], null, 'sum', calculatedValues); + + const aDataset = result.datasets.find(d => d.label === 'a'); + const bDataset = result.datasets.find(d => d.label === 'b'); + expect(aDataset.data[0]).toBe(2); // sum of a: 2+0+0 = 2 + expect(bDataset.data[0]).toBe(1); // sum of b: 0+0+1 = 1 + + const avgDataset = result.datasets.find(d => d.label === 'avg'); + // Expected: (sum_a + sum_b) / 2 = (2 + 1) / 2 = 1.5 + expect(avgDataset.data[0]).toBe(1.5); + }); + }); }); });