Chunk Entry Queries

SQLite has a default maximum number of variables when using IN queries. This causes SQLite to fail when too many entries match the initial sql query. See https://www.sqlite.org/limits.html#max_variable_number for more information.

Currently, sqlite3 version 4.2.0 uses sqlite binary 3.24.0, which has `SQLITE_MAXIMUM_VARIABLE_NUMBER = 999`. Upgrading to sqlite >= 3.32 would allow for that maximum number to be 32766, but that hasn't been done yet. Track progress of that upgrade in TryGhost/node-sqlite3#1340.

src/constants.ts

@@ -0,0 +1 @@
+export const SQLITE_MAXIMUM_VARIABLE_NUMBER = 999;

src/controller/EntryController.ts

@@ -5,7 +5,8 @@ import { Impression } from '../entity/Impression';
 import { DateRange } from '../entity/DateRange';
 import { Tag } from '../entity/Tag';
 import { ContentType } from '../enums';
-import { arrayify } from '../utils/arrayUtils';
+import { SQLITE_MAXIMUM_VARIABLE_NUMBER } from '../constants';
+import { arrayify, splitArray } from '../utils/arrayUtils';
 import {
     getOffsetDate,
     dateToSqliteTimestamp,
@@ -38,6 +39,7 @@ export class EntryController {
 
         const start = startDateOrDefault(httpQuery.start as string);
         const end = endDateOrDefault(httpQuery.end as string);
+        const tags = httpQuery.tags;
 
         let initialSqlQuery = this.entryRepo
             .createQueryBuilder('entry')
@@ -66,31 +68,16 @@ export class EntryController {
 
         const initialEntries = await initialSqlQuery.getMany();
 
-        let sqlQuery = this.entryRepo
-            .createQueryBuilder('entry')
-            .where('entry.id in (:...ids)', { ids: initialEntries.map((entry) => entry.id) })
-            .leftJoinAndMapMany(
-                'entry.dateRanges',
-                DateRange,
-                'dateRanges',
-                'entry.subjectDate >= dateRanges.start AND entry.subjectDate <= dateRanges.end',
+        const filteredEntries = (
+            await Promise.all(
+                splitArray(
+                    initialEntries.map((entry) => entry.id),
+                    SQLITE_MAXIMUM_VARIABLE_NUMBER,
+                ).map(async (ids) => {
+                    return await this.buildEntrySubquery(tags, ids).getMany();
+                }),
             )
-            .leftJoinAndMapOne(
-                'dateRanges.impression',
-                Impression,
-                'impression',
-                `impression.id = dateRanges.impressionId`,
-            )
-            .orderBy('entry.subjectDate');
-
-        const tags = request.query.tags;
-        if (tags) {
-            sqlQuery = sqlQuery.innerJoinAndSelect('dateRanges.tags', 'tag', 'tag.name IN (:...tags)', {
-                tags: arrayify(tags),
-            });
-        }
-
-        const filteredEntries = await sqlQuery.getMany();
+        ).reduce((acc, chunk) => acc.concat(chunk));
 
         const formattedEntries = filteredEntries.map((entry) => {
             return {
@@ -362,4 +349,31 @@ export class EntryController {
             return array.findIndex((other) => value.name == other.name) === index;
         });
     }
+
+    private buildEntrySubquery(tags, ids: number[]) {
+        let sqlQuery = this.entryRepo
+            .createQueryBuilder('entry')
+            .where('entry.id IN (:...ids)', { ids: ids })
+            .leftJoinAndMapMany(
+                'entry.dateRanges',
+                DateRange,
+                'dateRanges',
+                'entry.subjectDate >= dateRanges.start AND entry.subjectDate <= dateRanges.end',
+            )
+            .leftJoinAndMapOne(
+                'dateRanges.impression',
+                Impression,
+                'impression',
+                `impression.id = dateRanges.impressionId`,
+            )
+            .orderBy('entry.subjectDate');
+
+        if (tags) {
+            sqlQuery = sqlQuery.innerJoinAndSelect('dateRanges.tags', 'tag', 'tag.name IN (:...tags)', {
+                tags: arrayify(tags),
+            });
+        }
+
+        return sqlQuery;
+    }
 }

src/utils/arrayUtils.ts

@@ -1,3 +1,11 @@
 export const arrayify = (input: any[] | any): any[] => {
     return Array.isArray(input) ? input : [input];
 };
+
+export const splitArray = (array: any[], splitSize: number) => {
+    const splits = [];
+    for (let i = 0; i < array.length; i += splitSize) {
+        splits.push(array.slice(i, i + splitSize));
+    }
+    return splits;
+};