Skip to content

Commit

Permalink
Aggregate and display unknown values (#370)
Browse files Browse the repository at this point in the history
  • Loading branch information
macfarlandian committed Mar 25, 2021
1 parent aa40cef commit 3c41224
Show file tree
Hide file tree
Showing 34 changed files with 973 additions and 77 deletions.
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "TRC", "race_or_ethnicity": "AMERICAN_INDIAN_ALASKAN_NATIVE", "gender": "ALL", "age_bucket": "ALL", "total_population": "7"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "FTPFAR", "race_or_ethnicity": "ALL", "gender": "MALE", "age_bucket": "ALL", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "JRCC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "EXTERNAL_UNKNOWN", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPMND", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "40<", "total_population": "3"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "FTPMND", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "40<", "total_population": "0"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "GFC", "race_or_ethnicity": "HISPANIC", "gender": "ALL", "age_bucket": "ALL", "total_population": "0"}
Expand All @@ -17,7 +16,6 @@
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPMND", "race_or_ethnicity": "WHITE", "gender": "ALL", "age_bucket": "ALL", "total_population": "5"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "FTPFAR", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "40<", "total_population": "3"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "FTPMND", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "ALL", "total_population": "5"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "BTC", "race_or_ethnicity": "ALL", "gender": "EXTERNAL_UNKNOWN", "age_bucket": "ALL", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "DWCRC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "35-39", "total_population": "6"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPMND", "race_or_ethnicity": "AMERICAN_INDIAN_ALASKAN_NATIVE", "gender": "ALL", "age_bucket": "ALL", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPMND", "race_or_ethnicity": "BLACK", "gender": "ALL", "age_bucket": "ALL", "total_population": "0"}
Expand All @@ -27,8 +25,6 @@
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "DWCRC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "ALL", "total_population": "111"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "FTPMND", "race_or_ethnicity": "WHITE", "gender": "ALL", "age_bucket": "ALL", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "GFC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "35-39", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MRCC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "EXTERNAL_UNKNOWN", "total_population": "0"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "ALL", "race_or_ethnicity": "ALL", "gender": "EXTERNAL_UNKNOWN", "age_bucket": "ALL", "total_population": "7"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "DWCRC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "<25", "total_population": "9"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "JRCC", "race_or_ethnicity": "WHITE", "gender": "ALL", "age_bucket": "ALL", "total_population": "170"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "DWCRC", "race_or_ethnicity": "ALL", "gender": "MALE", "age_bucket": "ALL", "total_population": "2"}
Expand All @@ -37,7 +33,6 @@
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "TRC", "race_or_ethnicity": "WHITE", "gender": "ALL", "age_bucket": "ALL", "total_population": "10"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MRCC", "race_or_ethnicity": "HISPANIC", "gender": "ALL", "age_bucket": "ALL", "total_population": "2"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "GFC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "<25", "total_population": "0"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MRCC", "race_or_ethnicity": "ALL", "gender": "EXTERNAL_UNKNOWN", "age_bucket": "ALL", "total_population": "0"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "BTC", "race_or_ethnicity": "AMERICAN_INDIAN_ALASKAN_NATIVE", "gender": "ALL", "age_bucket": "ALL", "total_population": "15"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "DWCRC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "25-29", "total_population": "33"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "FTPFAR", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "25-29", "total_population": "5"}
Expand All @@ -50,7 +45,6 @@
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "NDSP", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "35-39", "total_population": "89"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "GFC", "race_or_ethnicity": "AMERICAN_INDIAN_ALASKAN_NATIVE", "gender": "ALL", "age_bucket": "ALL", "total_population": "2"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPMND", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "ALL", "total_population": "7"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "BTC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "EXTERNAL_UNKNOWN", "total_population": "0"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPFAR", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "ALL", "total_population": "5"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPMND", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "30-34", "total_population": "2"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "ALL", "race_or_ethnicity": "ALL", "gender": "MALE", "age_bucket": "ALL", "total_population": "661"}
Expand All @@ -62,12 +56,10 @@
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "TRC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "ALL", "total_population": "11"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "BTC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "40<", "total_population": "13"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "FTPFAR", "race_or_ethnicity": "BLACK", "gender": "ALL", "age_bucket": "ALL", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "JRCC", "race_or_ethnicity": "ALL", "gender": "EXTERNAL_UNKNOWN", "age_bucket": "ALL", "total_population": "2"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "JRCC", "race_or_ethnicity": "OTHER", "gender": "ALL", "age_bucket": "ALL", "total_population": "5"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "NDSP", "race_or_ethnicity": "AMERICAN_INDIAN_ALASKAN_NATIVE", "gender": "ALL", "age_bucket": "ALL", "total_population": "99"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPFAR", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "35-39", "total_population": "5"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "GFC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "ALL", "total_population": "9"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "NDSP", "race_or_ethnicity": "ALL", "gender": "EXTERNAL_UNKNOWN", "age_bucket": "ALL", "total_population": "0"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "DWCRC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "30-34", "total_population": "9"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MRCC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "35-39", "total_population": "17"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MRCC", "race_or_ethnicity": "ALL", "gender": "MALE", "age_bucket": "ALL", "total_population": "77"}
Expand All @@ -84,9 +76,7 @@
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "DWCRC", "race_or_ethnicity": "WHITE", "gender": "ALL", "age_bucket": "ALL", "total_population": "71"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "JRCC", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "ALL", "total_population": "257"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "JRCC", "race_or_ethnicity": "AMERICAN_INDIAN_ALASKAN_NATIVE", "gender": "ALL", "age_bucket": "ALL", "total_population": "88"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "NDSP", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "EXTERNAL_UNKNOWN", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "ALL", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "35-39", "total_population": "242"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "ALL", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "EXTERNAL_UNKNOWN", "total_population": "6"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPFAR", "race_or_ethnicity": "ALL", "gender": "ALL", "age_bucket": "<25", "total_population": "1"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MTPMND", "race_or_ethnicity": "ALL", "gender": "MALE", "age_bucket": "ALL", "total_population": "8"}
{"state_code": "US_ND", "date_of_stay": "2020-07-23", "facility": "MRCC", "race_or_ethnicity": "BLACK", "gender": "ALL", "age_bucket": "ALL", "total_population": "18"}
Expand Down
4 changes: 2 additions & 2 deletions spotlight-client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"@reach/router": "^1.3.4",
"@recidiviz/case-triage-components": "^0.2.2",
"@types/classnames": "^2.2.11",
"@types/d3-array": "^2.8.0",
"@types/d3-array": "^2.9.0",
"@types/d3-color": "^2.0.1",
"@types/d3-dsv": "^2.0.1",
"@types/d3-force": "^2.1.0",
Expand All @@ -44,7 +44,7 @@
"assert-never": "^1.2.1",
"change-case": "^4.1.2",
"classnames": "^2.2.6",
"d3-array": "^2.9.1",
"d3-array": "^2.12.0",
"d3-color": "^2.0.0",
"d3-dsv": "^2.0.0",
"d3-force": "^2.1.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,23 +20,30 @@ import React from "react";
import styled from "styled-components/macro";
import { colors } from "../UiLibrary";

const Wrapper = styled.div`
const Wrapper = styled.ol`
color: ${colors.caption};
font-size: ${rem(13)};
font-weight: 500;
line-height: 1.7;
list-style: decimal outside;
margin-top: ${rem(40)};
padding-left: 1em;
`;

const SmallDataDisclaimer = (): React.ReactElement => {
const Item = styled.li`
margin-bottom: 1em;
padding-left: 0.5em;
`;

/**
* Renders children as an ordered list of foot- or endnotes.
*/
const Notes: React.FC = ({ children }) => {
return (
<Wrapper>
Please always take note of the number of people associated with each
proportion presented here; in cases where the counts are especially low,
the proportion may not be statistically significant and therefore not
indicative of long-term trends.
{React.Children.map(children, (child) => child && <Item>{child}</Item>)}
</Wrapper>
);
};

export default SmallDataDisclaimer;
export default Notes;
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
// =============================================================================

export { default } from "./SmallDataDisclaimer";
export { default } from "./Notes";
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ import React, { useState } from "react";
import styled from "styled-components/macro";
import { ItemToHighlight, ProportionalBar } from "../charts";
import { DemographicCategoryRecords } from "../contentModels/types";
import SmallDataDisclaimer from "../SmallDataDisclaimer";
import VizControls, { VizControlsProps } from "../VizControls";
import VizNotes from "../VizNotes";

const CHART_HEIGHT = 165;

Expand Down Expand Up @@ -80,7 +80,7 @@ export default function BarChartPair({
highlighted={highlightedCategory}
setHighlighted={setHighlightedCategory}
/>
<SmallDataDisclaimer />
<VizNotes smallData />
</Wrapper>
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ import { useHighlightedItem } from "../charts/utils";
import DemographicsByCategoryMetric from "../contentModels/DemographicsByCategoryMetric";
import DemographicFilterSelect from "../DemographicFilterSelect";
import MetricVizControls from "../MetricVizControls";
import SmallDataDisclaimer from "../SmallDataDisclaimer";
import { animation, zIndex } from "../UiLibrary";
import VizNotes from "../VizNotes";
import withMetricHydrator from "../withMetricHydrator";

const bubbleChartHeight = 325;
Expand All @@ -54,7 +54,7 @@ const VizDemographicsByCategory: React.FC<VizDemographicsByCategoryProps> = ({
}) => {
const { highlighted, setHighlighted } = useHighlightedItem();

const { demographicView, dataSeries } = metric;
const { demographicView, dataSeries, unknowns } = metric;

const [chartContainerStyles, setChartContainerStyles] = useSpring(() => ({
from: { height: bubbleChartHeight },
Expand Down Expand Up @@ -130,7 +130,7 @@ const VizDemographicsByCategory: React.FC<VizDemographicsByCategoryProps> = ({
</ChartWrapper>
))}
</animated.div>
<SmallDataDisclaimer />
<VizNotes smallData unknowns={unknowns} />
</>
)}
</Measure>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import type HistoricalPopulationBreakdownMetric from "../contentModels/Historica
import DemographicFilterSelect from "../DemographicFilterSelect";
import MetricVizControls from "../MetricVizControls";
import { Dropdown } from "../UiLibrary";
import VizNotes from "../VizNotes";
import withMetricHydrator from "../withMetricHydrator";

const VizHistoricalPopulationBreakdown: React.FC<{
Expand Down Expand Up @@ -73,6 +74,7 @@ const VizHistoricalPopulationBreakdown: React.FC<{
defaultRangeEnd={defaultRangeEnd}
defaultRangeStart={defaultRangeStart}
/>
<VizNotes unknowns={metric.unknowns} />
</>
);

Expand Down
101 changes: 101 additions & 0 deletions spotlight-client/src/VizNotes/VizNotes.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
// Recidiviz - a data platform for criminal justice reform
// Copyright (C) 2021 Recidiviz, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
// =============================================================================

import { render, screen } from "@testing-library/react";
import React from "react";
import VizNotes from "./VizNotes";

test("all notes", () => {
render(
<VizNotes
smallData
unknowns={{ gender: 1, ageBucket: 0, raceOrEthnicity: 0 }}
/>
);

const items = screen.getAllByRole("listitem");
expect(items[0]).toHaveTextContent("counts are especially low");
expect(items[1]).toHaveTextContent(
"age, gender, or race/ethnicity is not reported"
);
});

test("small data only", () => {
render(<VizNotes smallData />);

expect(screen.getByRole("listitem")).toHaveTextContent(
"counts are especially low"
);
});

test("unknowns only", () => {
render(
<VizNotes unknowns={{ gender: 1, ageBucket: 0, raceOrEthnicity: 0 }} />
);

expect(screen.getByRole("listitem")).toHaveTextContent(
"age, gender, or race/ethnicity is not reported"
);
});

test("format single unknowns", () => {
render(
<VizNotes unknowns={{ gender: 1, ageBucket: 4, raceOrEthnicity: 0 }} />
);

expect(screen.getByRole("listitem")).toHaveTextContent(
"age group (4), gender (1)."
);
});

test("format unknowns by date", () => {
render(
<VizNotes
unknowns={[
{
date: new Date(2021, 0),
unknowns: { gender: 1, ageBucket: 0, raceOrEthnicity: 1 },
},
{
date: new Date(2021, 1),
unknowns: { gender: 2, ageBucket: 0, raceOrEthnicity: 0 },
},
]}
/>
);

expect(screen.getByRole("listitem")).toHaveTextContent(
"gender (1), race or ethnicity (1) for Jan 1 2021; gender (2) for Feb 1 2021."
);
});

test("format unknowns by cohort", () => {
render(
<VizNotes
unknowns={[
{
cohort: 2012,
unknowns: { gender: 1, ageBucket: 2, raceOrEthnicity: 3 },
},
]}
/>
);

expect(screen.getByRole("listitem")).toHaveTextContent(
"age group (2), gender (1), race or ethnicity (3) for the 2012 cohort."
);
});
92 changes: 92 additions & 0 deletions spotlight-client/src/VizNotes/VizNotes.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Recidiviz - a data platform for criminal justice reform
// Copyright (C) 2021 Recidiviz, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
// =============================================================================

import { format } from "date-fns";
import React from "react";
import { ValuesType } from "utility-types";
import { UnknownCounts, Unknowns } from "../contentModels/types";
import { getDemographicViewLabel } from "../demographics";
import { DemographicFieldKeyList } from "../metricsApi";
import Notes from "../Notes";
import { formatAsNumber } from "../utils";

function formatUnknownCounts(unknowns: UnknownCounts) {
const parts: string[] = [];

DemographicFieldKeyList.forEach((key) => {
const value = unknowns[key];
if (!value) return;

parts.push(
`${getDemographicViewLabel(key).toLowerCase()} (${formatAsNumber(value)})`
);
});

return parts.join(", ");
}

function formatUnknowns(unknowns: Unknowns) {
if (Array.isArray(unknowns)) {
// Typescript freaks out over unions of array types,
// but this assertion is functionally identical
return (unknowns as ValuesType<typeof unknowns>[])
.map((entry) => {
const formattedCounts = formatUnknownCounts(entry.unknowns);
let label: string;

if ("date" in entry) {
label = format(entry.date, "MMM d y");
} else {
label = `the ${entry.cohort} cohort`;
}

return `${formattedCounts} for ${label}`;
})
.join("; ");
}
return formatUnknownCounts(unknowns);
}

type VizNotesProps = {
smallData?: boolean;
unknowns?: Unknowns;
};

const VizNotes: React.FC<VizNotesProps> = ({ smallData, unknowns }) => {
return (
<Notes>
{smallData && (
<>
Please always take note of the number of people associated with each
proportion presented here; in cases where the counts are especially
low, the proportion may not be statistically significant and therefore
not indicative of long-term trends.
</>
)}
{unknowns && (
<>
This data includes some individuals for whom age, gender, or
race/ethnicity is not reported. These individuals count toward the
total but are excluded from demographic breakdown views. Unknown
values comprise: {formatUnknowns(unknowns)}.
</>
)}
</Notes>
);
};

export default VizNotes;
Loading

0 comments on commit 3c41224

Please sign in to comment.