Merge pull request #66 from mit-jp/aesthetic-changes

update data normalization calculations for map graphic to align with report card
mit-jp · Nov 6, 2023 · c47e5ce · c47e5ce
2 parents 8bcab3d + aecbad8
commit c47e5ce
Show file tree

Hide file tree

Showing 4 changed files with 138 additions and 49 deletions.
diff --git a/backend/src/dao/data_dao.rs b/backend/src/dao/data_dao.rs
@@ -86,7 +86,7 @@ impl<'c> Table<'c, Data> {
                     AND start_date = entry.start_date
                     AND end_date = entry.end_date
             ),
-            CASE WHEN (
+            CASE (
                 SELECT
                     value
                 FROM
@@ -97,22 +97,37 @@ impl<'c> Table<'c, Data> {
                     AND source = entry.source
                     AND start_date = entry.start_date
                     AND end_date = entry.end_date
-            ) IS NULL THEN NULL ELSE
-            percent_rank(
-                (
+            ) 
+            WHEN NULL THEN NULL 
+            WHEN 0 THEN 0
+            ELSE (
+                SELECT percent_rank FROM
+                (SELECT *,
+                  percent_rank() OVER (ORDER BY CASE WHEN entry.invert_normalized THEN -value ELSE value END) AS percent_rank
+                FROM 
+                    (SELECT * FROM data
+                        WHERE 
+                            dataset = entry.dataset
+                            AND source = entry.source
+                            AND start_date = entry.start_date
+                            AND end_date = entry.end_date
+                            AND value != 0
+                    UNION
                     SELECT
-                        CASE WHEN entry.invert_normalized THEN -value ELSE value END as value
-                    FROM
-                        data
-                    WHERE
-                        id = $1
-                        AND dataset = entry.dataset
-                        AND source = entry.source
-                        AND start_date = entry.start_date
-                        AND end_date = entry.end_date
-                )
-            ) within GROUP (
-                ORDER BY CASE WHEN entry.invert_normalized THEN -value ELSE value END
+                      0 AS dataset,
+                      0 AS source,
+                      '2000-01-01' AS start_date,
+                      '2000-01-01' AS end_date,
+                      0 AS value,
+                      0 AS id,
+                      0 AS geography_type) 
+                            AS state_data) AS percents
+                WHERE 
+                    id = $1
+                    AND dataset = entry.dataset
+                    AND source = entry.source
+                    AND start_date = entry.start_date
+                    AND end_date = entry.end_date
             )
         END
         FROM
@@ -206,35 +221,50 @@ impl<'c> Table<'c, Data> {
                     AND start_date = entry.start_date
                     AND end_date = entry.end_date
             ),
-            CASE WHEN (
+            CASE (
                 SELECT
                     value
                 FROM
                     (SELECT * FROM data
                     WHERE FLOOR(id / 1000) = FLOOR($1 / 1000)) AS state_data
                 WHERE
-                    id =($1)
+                    id = $1
                     AND dataset = entry.dataset
                     AND source = entry.source
                     AND start_date = entry.start_date
                     AND end_date = entry.end_date
-            ) IS NULL THEN NULL ELSE
-            percent_rank(
-                (
+            ) 
+            WHEN NULL THEN NULL 
+            WHEN 0 THEN 0
+            ELSE (
+                SELECT percent_rank FROM
+                (SELECT *,
+                  percent_rank() OVER (ORDER BY CASE WHEN entry.invert_normalized THEN -value ELSE value END) AS percent_rank
+                FROM 
+                    (SELECT * FROM data
+                        WHERE 
+                            FLOOR(id / 1000) = FLOOR($1 / 1000)
+                            AND dataset = entry.dataset
+                            AND source = entry.source
+                            AND start_date = entry.start_date
+                            AND end_date = entry.end_date
+                            AND value != 0
+                    UNION
                     SELECT
-                        CASE WHEN entry.invert_normalized THEN -value ELSE value END as value
-                    FROM
-                        (SELECT * FROM data
-                        WHERE FLOOR(id / 1000) = FLOOR($1 / 1000)) AS state_data
-                    WHERE
-                        id =($1)
-                        AND dataset = entry.dataset
-                        AND source = entry.source
-                        AND start_date = entry.start_date
-                        AND end_date = entry.end_date
-                )
-            ) within GROUP (
-                ORDER BY CASE WHEN entry.invert_normalized THEN -value ELSE value END
+                      0 AS dataset,
+                      0 AS source,
+                      '2000-01-01' AS start_date,
+                      '2000-01-01' AS end_date,
+                      0 AS value,
+                      0 AS id,
+                      0 AS geography_type) 
+                            AS state_data) AS percents
+                WHERE 
+                    id = $1
+                    AND dataset = entry.dataset
+                    AND source = entry.source
+                    AND start_date = entry.start_date
+                    AND end_date = entry.end_date
             )
         END
         FROM

diff --git a/frontend/src/DataProcessor.test.ts b/frontend/src/DataProcessor.test.ts
@@ -6,19 +6,34 @@ const DATA = Map<number, Data2>([
     [
         1,
         [
+            [0, 0],
             [1, 1],
             [2, 2],
             [3, 3],
+            [4, 4],
         ],
     ],
 ])
 const DATA_2 = Map<number, Data2>([
     [
         2,
         [
+            [0, 0],
             [1, 100],
             [2, 20],
             [3, 30],
+            [4, 40],
+        ],
+    ],
+])
+const DATA_3 = Map<number, Data2>([
+    [
+        3,
+        [
+            [0, 0],
+            [1, 1],
+            [2, 2],
+            [3, 3],
         ],
     ],
 ])
@@ -31,9 +46,11 @@ test('it returns a single dataset, not normalized', () => {
         })
     ).toEqual(
         Map([
+            [0, 0],
             [1, 1],
             [2, 2],
             [3, 3],
+            [4, 4],
         ])
     )
 })
@@ -47,9 +64,11 @@ test('it returns a single dataset, normalized', () => {
         })
     ).toEqual(
         Map([
-            [1, 0],
+            [0, 0],
+            [1, 0.25],
             [2, 0.5],
-            [3, 1],
+            [3, 0.75],
+            [4, 1],
         ])
     )
 })
@@ -65,9 +84,11 @@ test('it returns two datasets, added, if they are both selected', () => {
         })
     ).toEqual(
         Map([
+            [0, 0],
             [1, 101],
             [2, 22],
             [3, 33],
+            [4, 44],
         ])
     )
 })
@@ -84,9 +105,11 @@ test('it returns two datasets, normalized then averaged, if they are both select
         })
     ).toEqual(
         Map([
-            [1, 0.5],
-            [2, 0.25],
-            [3, 0.75],
+            [0, 0],
+            [1, 0.625],
+            [2, 0.375],
+            [3, 0.625],
+            [4, 0.875],
         ])
     )
 })
@@ -137,9 +160,11 @@ test('it weights datasets when normalizing', () => {
         })
     ).toEqual(
         Map([
-            [1, 0.75],
-            [2, 0.125],
-            [3, 0.625],
+            [0, 0],
+            [1, 0.8125],
+            [2, 0.3125],
+            [3, 0.5625],
+            [4, 0.8125],
         ])
     )
 })
@@ -155,9 +180,11 @@ test("it doesn't weight datasets when not normalizing", () => {
         })
     ).toEqual(
         Map([
+            [0, 0],
             [1, 101],
             [2, 22],
             [3, 33],
+            [4, 44],
         ])
     )
 })
@@ -171,23 +198,26 @@ test('it filters geoIds', () => {
         })
     ).toEqual(
         Map([
+            [0, 0],
             [1, 1],
             [3, 3],
+            [4, 4],
         ])
     )
 })
 
 test('it filters by geoIds, then normalizes with respect to only those ids', () => {
     expect(
         DataProcessor({
-            data: DATA,
-            params: [{ mapId: 1, weight: 1, invertNormalized: false }],
+            data: DATA_3,
+            params: [{ mapId: 3, weight: 1, invertNormalized: false }],
             filter: (geoId) => geoId !== 3,
             normalize: true,
         })
     ).toEqual(
         Map([
-            [1, 0],
+            [0, 0],
+            [1, 0.5],
             [2, 1],
         ])
     )

diff --git a/frontend/src/DataProcessor.ts b/frontend/src/DataProcessor.ts
@@ -1,4 +1,4 @@
-import { scaleSequentialQuantile, sum } from 'd3'
+import { sum } from 'd3'
 import { Map, Set } from 'immutable'
 import { Data2 } from './MapApi'
 import { MapVisualizationId } from './MapVisualization'
@@ -18,10 +18,38 @@ const normalizeData = (params: Params, totalWeight: number, valueByGeoId: Map<nu
         ? valueByGeoId.map((value) => -1 * value)
         : valueByGeoId
 
-    const weightedPercentileScale = scaleSequentialQuantile(
-        normalizedValueByGeoId.valueSeq(),
-        (quantile) => quantile * weight
+    let normalizedValueByGeoIdNonZero = Map<number, number>(
+        JSON.parse(JSON.stringify(Array.from(normalizedValueByGeoId)))
     )
+    // eslint-disable-next-line no-restricted-syntax
+    for (const entry of normalizedValueByGeoIdNonZero) {
+        if (entry[1] === 0) {
+            normalizedValueByGeoIdNonZero = normalizedValueByGeoIdNonZero.delete(entry[0])
+        }
+    }
+    normalizedValueByGeoIdNonZero = normalizedValueByGeoIdNonZero.set(0, 0)
+    const valueListNonZero = normalizedValueByGeoIdNonZero
+        .valueSeq()
+        .toArray()
+        .sort((a, b) => a - b)
+
+    const weightedPercentileScale = (value: number) => {
+        if (value === 0) {
+            return 0
+        }
+        // eslint-disable-next-line no-plusplus
+        for (let i = 0, l = valueListNonZero.length; i < l; i++) {
+            if (value <= valueListNonZero[i]) {
+                if (value !== valueListNonZero[i - 1]) {
+                    i +=
+                        (value - valueListNonZero[i - 1]) /
+                        (valueListNonZero[i] - valueListNonZero[i - 1])
+                }
+                return ((i - 1) / (l - 1)) * weight
+            }
+        }
+        return 1
+    }
     return normalizedValueByGeoId.map((value) => weightedPercentileScale(value))
 }
 

diff --git a/git-credential-manager b/git-credential-manager