Skip to content

Commit

Permalink
impute relative to data, not current date
Browse files Browse the repository at this point in the history
  • Loading branch information
@macfarlandian
macfarlandian committed Jun 2, 2021
1 parent 192f039 commit 5e12fce
Show file tree
Hide file tree
Showing 6 changed files with 116 additions and 222 deletions.
9 changes: 4 additions & 5 deletions spotlight-client/src/VizHistoricalPopulationBreakdown/VizHistoricalPopulationBreakdown.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
// =============================================================================

import { startOfMonth, sub } from "date-fns";
import { sub } from "date-fns";
import { observer } from "mobx-react-lite";
import React, { useState } from "react";
import { isWindowSizeId, WindowedTimeSeries, WindowSizeId } from "../charts";
Expand All @@ -31,10 +31,9 @@ const VizHistoricalPopulationBreakdown: React.FC<{
}> = ({ metric }) => {
const [windowSizeId, setWindowSizeId] = useState<WindowSizeId>("20");

let defaultRangeEnd = startOfMonth(new Date());
if (!metric.dataIncludesCurrentMonth) {
defaultRangeEnd = sub(defaultRangeEnd, { months: 1 });
}
if (!metric.latestMonthInData) return null;

const defaultRangeEnd = metric.latestMonthInData;

let defaultRangeStart: Date | undefined;
if (windowSizeId !== "custom") {
Expand Down
47 changes: 4 additions & 43 deletions spotlight-client/src/contentModels/HistoricalPopulationBreakdownMetric.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
// =============================================================================

import { isEqual } from "date-fns";
import { advanceTo, clear } from "jest-date-mock";
import { runInAction, when } from "mobx";
import {
createDemographicCategories,
Expand Down Expand Up @@ -74,10 +73,6 @@ beforeEach(() => {
mockedFetchAndTransformMetric.mockResolvedValue([...mockData]);
});

afterEach(() => {
clear();
});

const getMetric = async () => {
const metric = new HistoricalPopulationBreakdownMetric({
...contentFixture.metrics.PrisonPopulationHistorical,
Expand Down Expand Up @@ -135,6 +130,10 @@ test("fills in missing data", async () => {
);
}
});

// records are sorted and span 240 months
expect(series[0].date).toEqual(new Date(2000, 10, 1));
expect(series[series.length - 1].date).toEqual(new Date(2020, 9, 1));
});
}
});
Expand All @@ -143,44 +142,6 @@ test("fills in missing data", async () => {
expect.hasAssertions();
});

test("imputed data does not include the current month", async () => {
// later than most recent month present in the fixture
advanceTo(new Date(2020, 11, 10));
const currentMonth = new Date(2020, 11, 1);

const metric = await getMetric();

reactImmediately(() => {
const currentMonthRecords = metric.records?.filter((record) =>
isEqual(record.date, currentMonth)
);

expect(currentMonthRecords?.length).toBe(0);

expect(metric.dataIncludesCurrentMonth).toBe(false);
});
expect.hasAssertions();
});

test("imputed data includes current month", async () => {
// most recent month present in the fixture
advanceTo(new Date(2020, 9, 10));
const currentMonth = new Date(2020, 9, 1);

const metric = await getMetric();

reactImmediately(() => {
const currentMonthRecords = metric.records?.filter((record) =>
isEqual(record.date, currentMonth)
);

expect(currentMonthRecords?.length).toBe(1);

expect(metric.dataIncludesCurrentMonth).toBe(true);
});
expect.hasAssertions();
});

test("no unknowns", async () => {
const metric = await getMetric();

Expand Down
112 changes: 57 additions & 55 deletions spotlight-client/src/contentModels/HistoricalPopulationBreakdownMetric.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
// =============================================================================

import { ascending, groups, sum } from "d3-array";
import { isSameDay, startOfMonth } from "date-fns";
import { isValid, max } from "date-fns";
import { computed, makeObservable, observable, runInAction } from "mobx";
import { DataSeries } from "../charts";
import {
Expand All @@ -38,11 +38,11 @@ import { UnknownsByDate } from "./types";

const EXPECTED_MONTHS = 240; // 20 years

function dataIncludesCurrentMonth(
records: HistoricalPopulationBreakdownRecord[]
) {
const thisMonth = startOfMonth(new Date());
return records.some((record) => isSameDay(record.date, thisMonth));
function latestMonthInData(records: HistoricalPopulationBreakdownRecord[]) {
// JS dates provide a month index, whereas records use calendar months
const latestDate = max(records.map((r) => r.date));
// an empty record set can result in an invalid date, which will break things
return isValid(latestDate) ? latestDate : undefined;
}

/**
Expand All @@ -54,16 +54,16 @@ function dataIncludesCurrentMonth(
*/
function getMissingMonthsForSeries({
demographicFields,
includeCurrentMonth,
end,
records,
}: {
demographicFields: DemographicFields;
includeCurrentMonth: boolean;
end: Date;
records: HistoricalPopulationBreakdownRecord[];
}): HistoricalPopulationBreakdownRecord[] {
const missingMonths = getMissingMonths({
end,
expectedMonths: EXPECTED_MONTHS,
includeCurrentMonth,
records: records.map(({ date }) => ({
year: date.getFullYear(),
monthIndex: date.getMonth(),
Expand All @@ -79,15 +79,15 @@ function getMissingMonthsForSeries({

export default class HistoricalPopulationBreakdownMetric extends Metric<HistoricalPopulationBreakdownRecord> {
// UI needs to know this in order to configure proper viewing window
dataIncludesCurrentMonth?: boolean;
latestMonthInData?: Date;

constructor(
props: BaseMetricConstructorOptions<HistoricalPopulationBreakdownRecord>
) {
super(props);

makeObservable(this, {
dataIncludesCurrentMonth: observable,
latestMonthInData: observable,
dataSeries: computed,
unknowns: computed,
});
Expand All @@ -96,56 +96,58 @@ export default class HistoricalPopulationBreakdownMetric extends Metric<Historic
async fetchAndTransform(): Promise<HistoricalPopulationBreakdownRecord[]> {
const transformedData = await super.fetchAndTransform();

// if the current month is completely missing from data, we will assume it is
// actually missing due to reporting lag. But if any record contains it, we will
// assume that it should be replaced with an empty record when it is missing
const includeCurrentMonth = dataIncludesCurrentMonth(transformedData);
runInAction(() => {
this.dataIncludesCurrentMonth = includeCurrentMonth;
});

const missingRecords: HistoricalPopulationBreakdownRecord[] = [];

// isolate each data series and impute any missing records
DemographicViewList.forEach((demographicView) => {
if (demographicView === "nofilter") return;
// our historical window ends at the latest date seen in the data;
// we won't impute anything more recent than that
const end = latestMonthInData(transformedData);

const recordsForDemographicView = transformedData.filter(
recordIsTotalByDimension(demographicView)
);
if (end) {
runInAction(() => {
this.latestMonthInData = end;
});

const categories = this.getDemographicCategories(demographicView);
categories.forEach(({ identifier }) => {
let recordsForCategory;
if (demographicView !== "total") {
recordsForCategory = recordsForDemographicView.filter(
(record) => record[demographicView] === identifier
);
} else {
recordsForCategory = recordsForDemographicView;
}
missingRecords.push(
...getMissingMonthsForSeries({
records: recordsForCategory,
includeCurrentMonth,
demographicFields: {
raceOrEthnicity:
demographicView === "raceOrEthnicity"
? (identifier as RaceIdentifier)
: "ALL",
gender:
demographicView === "gender"
? (identifier as GenderIdentifier)
: "ALL",
ageBucket:
demographicView === "ageBucket"
? (identifier as AgeIdentifier)
: "ALL",
},
})
// isolate each data series and impute any missing records
DemographicViewList.forEach((demographicView) => {
if (demographicView === "nofilter") return;

const recordsForDemographicView = transformedData.filter(
recordIsTotalByDimension(demographicView)
);

const categories = this.getDemographicCategories(demographicView);
categories.forEach(({ identifier }) => {
let recordsForCategory;
if (demographicView !== "total") {
recordsForCategory = recordsForDemographicView.filter(
(record) => record[demographicView] === identifier
);
} else {
recordsForCategory = recordsForDemographicView;
}
missingRecords.push(
...getMissingMonthsForSeries({
records: recordsForCategory,
end,
demographicFields: {
raceOrEthnicity:
demographicView === "raceOrEthnicity"
? (identifier as RaceIdentifier)
: "ALL",
gender:
demographicView === "gender"
? (identifier as GenderIdentifier)
: "ALL",
ageBucket:
demographicView === "ageBucket"
? (identifier as AgeIdentifier)
: "ALL",
},
})
);
});
});
});
}

transformedData.push(...missingRecords);

Expand Down
69 changes: 6 additions & 63 deletions spotlight-client/src/contentModels/SupervisionSuccessRateMetric.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import { csvParse } from "d3-dsv";
import downloadjs from "downloadjs";
import JsZip from "jszip";
import { advanceTo, clear } from "jest-date-mock";
import { runInAction, when } from "mobx";
import { stripHtml } from "string-strip-html";
import { DemographicView } from "../demographics";
Expand Down Expand Up @@ -69,15 +68,6 @@ async function getPopulatedMetric() {
return metric;
}

beforeEach(() => {
// last month in data fixture
advanceTo(new Date(2020, 6, 2));
});

afterEach(() => {
clear();
});

describe("cohort data", () => {
test("total", async () => {
const metric = await getPopulatedMetric();
Expand Down Expand Up @@ -155,61 +145,14 @@ describe("cohort data", () => {
);
});

// does not include current month, as set via jest date mock
expect(cohortRecords).toEqual(
expect.not.arrayContaining([
expect.objectContaining({ year: 2020, month: 7 }),
])
);
// as a result, the start should be shifted back a month
expect(cohortRecords).toEqual(
expect.arrayContaining([
expect.objectContaining({ year: 2017, month: 7 }),
])
// records should be sorted and span 36 months,
// ending with the latest date present in the data
expect(cohortRecords[0]).toEqual(
expect.objectContaining({ year: 2016, month: 10 })
);
});

expect.hasAssertions();
});

test("imputes missing cohorts with current month", async () => {
mockedFetchAndTransformMetric.mockResolvedValueOnce([
{
year: 2020,
month: 7,
district: "ALL",
rateNumerator: 2,
rateDenominator: 2,
rate: 1,
},
]);

const metric = await getPopulatedMetric();

reactImmediately(() => {
const cohortRecords = metric.cohortRecords as SupervisionSuccessRateMonthlyRecord[];
expect(cohortRecords.length).toEqual(36);
cohortRecords.forEach((record) => {
if (record.year !== 2020 && record.month !== 7)
expect(record).toEqual(
expect.objectContaining({
rate: 0,
rateDenominator: 0,
rateNumerator: 0,
})
);
});
// includes current month, as set via jest date mock
expect(cohortRecords).toEqual(
expect.arrayContaining([
expect.objectContaining({ year: 2020, month: 7 }),
])
);
// as a result, the start is shifted forward a month
expect(cohortRecords).toEqual(
expect.not.arrayContaining([
expect.objectContaining({ year: 2017, month: 7 }),
])
expect(cohortRecords[cohortRecords.length - 1]).toEqual(
expect.objectContaining({ year: 2019, month: 9 })
);
});

Expand Down
Loading

0 comments on commit 5e12fce

Please sign in to comment.