forked from hackforla/311-data
-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
499 lines (452 loc) · 17.9 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
/* eslint-disable */
import React from 'react';
import { connect } from 'react-redux';
import withStyles from '@mui/styles/withStyles';
import {
getDbRequest,
getDbRequestSuccess,
getDataRequest,
getDataRequestSuccess,
updateDateRanges,
} from '@reducers/data';
import {
updateStartDate,
updateEndDate,
updateNcId,
updateRequestTypes,
} from '@reducers/filters';
import { updateMapPosition } from '@reducers/ui';
import { trackMapExport } from '@reducers/analytics';
import { INTERNAL_DATE_SPEC } from '../common/CONSTANTS';
import { getTypeIdFromTypeName } from '@utils';
import LoadingModal from '../Loading/LoadingModal';
import FunFactCard from '@components/Loading/FunFactCard';
import CookieNotice from '../main/CookieNotice';
import Map from './Map';
import moment from 'moment';
import ddbh from '@utils/duckDbHelpers.js';
import DbContext from '@db/DbContext';
import AcknowledgeModal from '../Loading/AcknowledgeModal';
// We make API requests on a per-day basis. On average, there are about 4k
// requests per day, so 10k is a large safety margin.
const REQUEST_LIMIT = 10000;
const styles = (theme) => ({
root: {
height: `calc(100vh - ${theme.header.height} - ${theme.footer.height})`,
},
});
class MapContainer extends React.Component {
// Note: 'this.context' is defined using the static contextType property
// static contextType assignment allows MapContainer to access values provided by DbContext.Provider
static contextType = DbContext;
constructor(props) {
super(props);
this.state = {
ncCounts: null,
ccCounts: null,
position: props.position,
lastUpdated: props.lastUpdated,
selectedTypes: this.getSelectedTypes(),
acknowledgeModalShown: false,
isTableLoading: false,
};
// We store the raw requests from the API call here, but eventually they aremap/inde
// converted and stored in the Redux store.
this.rawRequests = [];
this.isSubscribed = null;
this.initialState = props.initialState;
this.startTime = 0;
this.endTime = 0;
}
createRequestsTable = async () => {
this.setState({ isTableLoading: true });
const { conn, tableNameByYear } = this.context;
const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store
const year = moment(startDate).year(); // extrac the year
const datasetFileName = `requests${year}.parquet`;
// Create the year data table if not exist already
const createSQL =
`CREATE TABLE IF NOT EXISTS ${tableNameByYear} AS SELECT * FROM "${datasetFileName}"`; // query from parquet
const startTime = performance.now(); // start the time tracker
try {
await conn.query(createSQL);
const endTime = performance.now() // end the timer
console.log(`Dataset registration & table creation (by year) time: ${Math.floor(endTime - startTime)} ms.`);
} catch (error) {
console.error("Error in creating table or registering dataset:", error);
} finally {
this.setState({ isTableLoading: false});
}
};
async componentDidMount(props) {
this.isSubscribed = true;
this.processSearchParams();
await this.createRequestsTable();
await this.setData();
}
async componentDidUpdate(prevProps) {
const { activeMode, pins, startDate, endDate } = this.props;
// create conditions to check if year or startDate or endDate changed
const yearChanged = moment(prevProps.startDate).year() !== moment(startDate).year();
const startDateChanged = prevProps.startDate !== startDate;
const endDateChanged = prevProps.endDate !== endDate;
// Check that endDate is not null since we only want to retrieve data
// when both the startDate and endDate are selected.
const didDateRangeChange = (yearChanged || startDateChanged || endDateChanged) && endDate !== null;
if (
prevProps.activeMode !== activeMode ||
prevProps.pins !== pins ||
didDateRangeChange
) {
await this.createRequestsTable();
await this.setData();
}
}
async componentWillUnmount() {
this.isSubscribed = false;
}
processSearchParams = () => {
// Dispatch to edit Redux store with url search params
const {
dispatchUpdateStartDate,
dispatchUpdateEndDate,
dispatchUpdateTypesFilter,
} = this.props;
// Filter requests on time
const dateFormat = 'YYYY-MM-DD';
// TODO: Check if endDate > startDate
if (
moment(this.initialState.startDate, 'YYYY-MM-DD', true).isValid() &&
moment(this.initialState.endDate, 'YYYY-MM-DD', true).isValid()
) {
const formattedStart = moment(this.initialState.startDate).format(
dateFormat
);
const formattedEnd = moment(this.initialState.endDate).format(dateFormat);
if (formattedStart <= formattedEnd) {
dispatchUpdateStartDate(formattedStart);
dispatchUpdateEndDate(formattedEnd);
}
}
for (let request_id = 1; request_id < 13; request_id++) {
if (this.initialState[`rtId${request_id}`] == 'false') {
dispatchUpdateTypesFilter(request_id);
}
}
};
/**
* Returns the non-overlapping date ranges of A before and after B.
* @param {string} startA The start date of range A in INTERNAL_DATE_SPEC format.
* @param {string} endA The end date of range A in INTERNAL_DATE_SPEC format.
* @param {string} startB The start date of range B in INTERNAL_DATE_SPEC format.
* @param {string} endB The end date of range B in INTERNAL_DATE_SPEC format.
* @returns An array of two elements: the first element is the non-overlapping
* range of A before B; the second is the non-overlapping range of A after B.
* Each element can be null if there is no non-overlappping range.
*/
getNonOverlappingRanges = (startA, endA, startB, endB) => {
var leftNonOverlap = null;
var rightNonOverlap = null;
const momentStartA = moment(startA);
const momentEndA = moment(endA);
const momentStartB = moment(startB);
const momentEndB = moment(endB);
// If date range A starts before date range B, then it has a subrange that
// does not overlap with B.
if (momentStartA < momentStartB) {
// For the left side, we want to choose the earlier of (startB, endA).
// If startB is earlier than endA, that means A and B overlap, so we
// subtract 1 day from startB, since it's already included in A.
const leftNonOverlapEnd =
momentStartB < momentEndA
? momentStartB.subtract(1, 'days')
: momentEndA;
leftNonOverlap = [startA, leftNonOverlapEnd.format(INTERNAL_DATE_SPEC)];
}
// If date range A ends after date range B, then it has a subrange that does
// not overlap with B.
if (momentEndB < momentEndA) {
var rightNonOverlapStart =
momentEndB < momentStartA ? momentStartA : momentEndB.add(1, 'days');
rightNonOverlap = [rightNonOverlapStart.format(INTERNAL_DATE_SPEC), endA];
}
return [leftNonOverlap, rightNonOverlap];
};
/**
* Returns the missing date ranges of a new date range against the existing
* date ranges in the Redux store.
*
* In our Redux store, we keep track of date ranges that we already have 311
* requests for. When the user changes the date range, we need to check
* whether we need to retrieve more data; if we do, we only want to pull the
* data from the date ranges that aren't already in the store.
*
* @param {string} startDate The start date in INTERNAL_DATE_SPEC format.
* @param {string} endDate The end date in INTERNAL_DATE_SPEC format.
* @returns An array of date ranges, where each date range is represented as
* an array of string start and end dates.
*/
getMissingDateRanges = (startDate, endDate) => {
const { dateRangesWithRequests } = this.props;
var missingDateRanges = [];
var currentStartDate = startDate;
var currentEndDate = endDate;
// Compare the input date range with each date range with requests, which
// are ordered chronologically from first to last. Every left non-overlapping
// date range (i.e., a portion of the input date range that comes before the
// existing date range with requests) is immediately added to the list of
// missing date ranges. Otherwise, if there is overlap on the left (i.e.,
// the input range is covered on the left side by the date range with
// requests), we push the start date for our input range forward to the end
// of the date range with requests. The process continues for every date
// range with requests.
// It stops when the input date range is covered on the right side.
for (let dateRange of dateRangesWithRequests.values()) {
const nonOverlappingRanges = this.getNonOverlappingRanges(
currentStartDate,
currentEndDate,
dateRange[0],
dateRange[1]
);
if (nonOverlappingRanges[0] !== null) {
missingDateRanges.push(nonOverlappingRanges[0]);
}
if (nonOverlappingRanges[1] === null) {
return missingDateRanges;
}
currentStartDate = nonOverlappingRanges[1][0];
currentEndDate = nonOverlappingRanges[1][1];
}
missingDateRanges.push([currentStartDate, currentEndDate]);
return missingDateRanges;
};
/**
* Returns the updated date ranges given the date ranges that we just pulled
* data for.
* @param {Array} newDateRanges The new date ranges that we just pulled data for.
* @returns The updated, complete array of date ranges for which we have data
* in the Redux store.
*/
resolveDateRanges = (newDateRanges) => {
const { dateRangesWithRequests } = this.props;
var allDateRanges = dateRangesWithRequests.concat(newDateRanges);
// Sort date ranges by startDate. Since newDateRanges was retrieved using
// getMissingDateRanges, there should be no overlapping date ranges in the
// allDateRanges.
const sortedDateRanges = allDateRanges.sort(function (
dateRangeA,
dateRangeB
) {
return moment(dateRangeA[0]) - moment(dateRangeB[0]);
});
var resolvedDateRanges = [];
var currentStart = null;
var currentEnd = null;
for (const dateRange of sortedDateRanges) {
if (currentStart === null) {
currentStart = dateRange[0];
currentEnd = dateRange[1];
continue;
}
// Check if the current date range is adjacent to the next date range.
if (
moment(currentEnd).add(1, 'days').valueOf() ===
moment(dateRange[0]).valueOf()
) {
// Extend the current date range to include the next date range.
currentEnd = dateRange[1];
} else {
resolvedDateRanges.push([currentStart, currentEnd]);
currentStart = dateRange[0];
currentEnd = dateRange[1];
}
}
if (currentStart !== null) {
resolvedDateRanges.push([currentStart, currentEnd]);
}
return resolvedDateRanges;
};
/**
* Gets all the dates within a given date range.
* @param {string} startDate A date in INTERNAL_DATE_SPEC format.
* @param {string} endDate A date in INTERNAL_DATE_SPEC format.
* @returns An array of string dates in INTERNAL_DATE_SPEC format, including
* the end date.
*/
getDatesInRange = (startDate, endDate) => {
var dateArray = [];
var currentDateMoment = moment(startDate, INTERNAL_DATE_SPEC);
const endDateMoment = moment(endDate, INTERNAL_DATE_SPEC);
while (currentDateMoment <= endDateMoment) {
dateArray.push(currentDateMoment.format(INTERNAL_DATE_SPEC));
currentDateMoment = currentDateMoment.add(1, 'days');
}
return dateArray;
};
// To handle cross-year date ranges, we check if the startDate and endDate year are the same year
// if same year, we simply query from that year's table
// if different years, we query both startDate year and endDate year, then union the result
async getAllRequests(startDate, endDate) {
const { conn } = this.context;
const startYear = moment(startDate).year();
const endYear = moment(endDate).year();
let selectSQL = '';
try {
if (startYear === endYear) {
// If the dates are within the same year, query that single year's table.
const tableName = `requests_${startYear}`;
selectSQL = `SELECT * FROM ${tableName} WHERE CreatedDate BETWEEN '${startDate}' AND '${endDate}'`;
} else {
// If the dates span multiple years, create two queries and union them.
const tableNameStartYear = `requests_${startYear}`;
const endOfStartYear = moment(startDate).endOf('year').format('YYYY-MM-DD');
const tableNameEndYear = `requests_${endYear}`;
const startOfEndYear = moment(endDate).startOf('year').format('YYYY-MM-DD');
selectSQL = `
(SELECT * FROM ${tableNameStartYear} WHERE CreatedDate BETWEEN '${startDate}' AND '${endOfStartYear}')
UNION ALL
(SELECT * FROM ${tableNameEndYear} WHERE CreatedDate BETWEEN '${startOfEndYear}' AND '${endDate}')
`;
}
const dataLoadStartTime = performance.now();
const requestsAsArrowTable = await conn.query(selectSQL);
const dataLoadEndTime = performance.now();
console.log(`Data loading time: ${Math.floor(dataLoadEndTime - dataLoadStartTime)} ms`);
const requests = ddbh.getTableData(requestsAsArrowTable);
const mapLoadEndTime = performance.now();
console.log(`Map loading time: ${Math.floor(mapLoadEndTime - dataLoadEndTime)} ms`);
return requests;
} catch (e) {
console.error("Error during database query execution:", e);
}
}
setData = async () => {
const { startDate, endDate, dispatchGetDbRequest, dispatchGetDataRequest } =
this.props;
const missingDateRanges = this.getMissingDateRanges(startDate, endDate);
if (missingDateRanges.length === 0) {
return;
}
dispatchGetDataRequest(); // set isMapLoading in redux stat.data to true
dispatchGetDbRequest(); // set isDbLoading in redux state.data to true
this.rawRequests = await this.getAllRequests(startDate, endDate);
if (this.isSubscribed) {
const {
dispatchGetDataRequestSuccess,
dispatchGetDbRequestSuccess,
dispatchUpdateDateRanges,
} = this.props;
const convertedRequests = this.convertRequests(this.rawRequests);
// load map features/requests upon successful map load
dispatchGetDataRequestSuccess(convertedRequests);
// set isDbLoading in redux state.data to false
dispatchGetDbRequestSuccess();
const newDateRangesWithRequests =
this.resolveDateRanges(missingDateRanges);
dispatchUpdateDateRanges(newDateRangesWithRequests);
}
};
convertRequests = (requests) =>
requests.map((request) => {
// Be careful, request properties are case-sensitive
return {
type: 'Feature',
properties: {
requestId: request.SRNumber,
typeId: getTypeIdFromTypeName(request.RequestType),
closedDate: request.ClosedDate,
// Store this in milliseconds so that it's easy to do date comparisons
// using Mapbox GL JS filters.
createdDateMs: moment(request.CreatedDate).valueOf(),
},
geometry: {
type: 'Point',
coordinates: [request.Longitude, request.Latitude],
},
};
});
// TODO: fix this
getSelectedTypes = () => {
const { requestTypes } = this.props;
// return Object.keys(requestTypes).filter(type => {
// return type !== 'All' && requestTypes[type]
// });
return requestTypes;
};
onClose = () => {
this.state.acknowledgeModalShown = true;
}
render() {
const {
position,
lastUpdated,
dispatchUpdateMapPosition,
dispatchTrackMapExport,
classes,
requests,
isMapLoading,
isDbLoading,
} = this.props;
const { ncCounts, ccCounts, selectedTypes, acknowledgeModalShown, isTableLoading } = this.state;
return (
<div className={classes.root}>
<Map
requests={requests}
ncCounts={ncCounts}
ccCounts={ccCounts}
position={position}
lastUpdated={lastUpdated}
updatePosition={dispatchUpdateMapPosition}
exportMap={dispatchTrackMapExport}
selectedTypes={selectedTypes}
initialState={this.initialState}
/>
<CookieNotice />
{(isDbLoading || isMapLoading || isTableLoading) ? (
<>
<LoadingModal />
<FunFactCard />
</>
) : (acknowledgeModalShown === false) ? (
<AcknowledgeModal onClose={this.onClose}/>
) : null}
</div>
);
}
}
const mapStateToProps = (state) => ({
pins: state.data.pins,
position: state.ui.map,
lastUpdated: state.metadata.lastPulledLocal,
activeMode: state.ui.map.activeMode,
requestTypes: state.filters.requestTypes,
startDate: state.filters.startDate,
endDate: state.filters.endDate,
requests: state.data.requests,
dateRangesWithRequests: state.data.dateRangesWithRequests,
isMapLoading: state.data.isMapLoading,
isDbLoading: state.data.isDbLoading,
});
const mapDispatchToProps = (dispatch) => ({
dispatchUpdateMapPosition: (position) =>
dispatch(updateMapPosition(position)),
dispatchTrackMapExport: () => dispatch(trackMapExport()),
dispatchGetDbRequest: () => dispatch(getDbRequest()),
dispatchGetDbRequestSuccess: (data) => dispatch(getDbRequestSuccess()),
dispatchGetDataRequest: () => dispatch(getDataRequest()),
dispatchGetDataRequestSuccess: (data) =>
dispatch(getDataRequestSuccess(data)),
dispatchUpdateDateRanges: (dateRanges) =>
dispatch(updateDateRanges(dateRanges)),
dispatchUpdateStartDate: (startDate) => dispatch(updateStartDate(startDate)),
dispatchUpdateEndDate: (endDate) => dispatch(updateEndDate(endDate)),
dispatchUpdateNcId: (id) => dispatch(updateNcId(id)),
dispatchUpdateTypesFilter: (type) => dispatch(updateRequestTypes(type)),
});
MapContainer.propTypes = {};
MapContainer.defaultProps = {};
// connect MapContainer to Redux store
export default connect(
mapStateToProps,
mapDispatchToProps
)(withStyles(styles)(MapContainer));