Skip to content

Commit

Permalink
fix: cuts time to get an app export (#3978)
Browse files Browse the repository at this point in the history
  • Loading branch information
@YazeedLoonat
YazeedLoonat committed Apr 3, 2024
1 parent c65dd2d commit cf27fab
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 126 deletions.
235 changes: 117 additions & 118 deletions api/src/services/application-csv-export.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ export const typeMap = {
fiveBdrm: 'Five Bedroom',
};

const NUMBER_TO_PAGINATE_BY = 500;

@Injectable()
export class ApplicationCsvExporterService
implements CsvExporterServiceInterface
Expand Down Expand Up @@ -94,6 +96,11 @@ export class ApplicationCsvExporterService
const applications = await this.prisma.applications.findMany({
select: {
id: true,
householdMember: {
select: {
id: true,
},
},
},
where: {
listingId: queryParams.listingId,
Expand All @@ -107,10 +114,13 @@ export class ApplicationCsvExporterService
queryParams.listingId,
);

// get maxHouseholdMembers or associated to the selected applications
const maxHouseholdMembers = await this.maxHouseholdMembers(
applications.map((application) => application.id),
);
// get maxHouseholdMembers associated to the selected applications
let maxHouseholdMembers = 0;
applications.forEach((app) => {
if (app.householdMember?.length > maxHouseholdMembers) {
maxHouseholdMembers = app.householdMember.length;
}
});

const csvHeaders = await this.getCsvHeaders(
maxHouseholdMembers,
Expand Down Expand Up @@ -138,131 +148,120 @@ export class ApplicationCsvExporterService
.join(',') + '\n',
);

for (let i = 0; i < applications.length / 1000 + 1; i++) {
// grab applications 1k at a time
const paginatedApplications =
await this.prisma.applications.findMany({
include: {
...view.csv,
demographics: queryParams.includeDemographics
? {
select: {
id: true,
createdAt: true,
updatedAt: true,
ethnicity: true,
gender: true,
sexualOrientation: true,
howDidYouHear: true,
race: true,
},
const promiseArray: Promise<string>[] = [];
for (let i = 0; i < applications.length; i += NUMBER_TO_PAGINATE_BY) {
promiseArray.push(
new Promise(async (resolve) => {
// grab applications NUMBER_TO_PAGINATE_BY at a time
const paginatedApplications =
await this.prisma.applications.findMany({
include: {
...view.csv,
demographics: queryParams.includeDemographics
? {
select: {
id: true,
createdAt: true,
updatedAt: true,
ethnicity: true,
gender: true,
sexualOrientation: true,
howDidYouHear: true,
race: true,
},
}
: false,
},
where: {
listingId: queryParams.listingId,
deletedAt: null,
},
skip: i,
take: NUMBER_TO_PAGINATE_BY,
});

let row = '';
paginatedApplications.forEach((app) => {
let preferences: ApplicationMultiselectQuestion[];
csvHeaders.forEach((header, index) => {
let multiselectQuestionValue = false;
let parsePreference = false;
let value = header.path.split('.').reduce((acc, curr) => {
// return preference/program as value for the format function to accept
if (multiselectQuestionValue) {
return acc;
}

if (parsePreference) {
// curr should equal the preference id we're pulling from
if (!preferences) {
preferences =
app.preferences as unknown as ApplicationMultiselectQuestion[];
}
parsePreference = false;
// there aren't typically many preferences, but if there, then a object map should be created and used
const preference = preferences.find(
(preference) =>
preference.multiselectQuestionId === curr,
);
multiselectQuestionValue = true;
return preference;
}

// sets parsePreference to true, for the next iteration
if (curr === 'preferences') {
parsePreference = true;
}

if (acc === null || acc === undefined) {
return '';
}

// handles working with arrays, e.g. householdMember.0.firstName
if (!isNaN(Number(curr))) {
const index = Number(curr);
return acc[index];
}
: false,
},
where: {
listingId: queryParams.listingId,
deletedAt: null,
},
skip: i * 1000,
take: 1000,
});

// now loop over applications and write them to file
paginatedApplications.forEach((app) => {
let row = '';
let preferences: ApplicationMultiselectQuestion[];
csvHeaders.forEach((header, index) => {
let multiselectQuestionValue = false;
let parsePreference = false;
let value = header.path.split('.').reduce((acc, curr) => {
// return preference/program as value for the format function to accept
if (multiselectQuestionValue) {
return acc;
}

if (parsePreference) {
// curr should equal the preference id we're pulling from
if (!preferences) {
preferences =
app.preferences as unknown as ApplicationMultiselectQuestion[];
return acc[curr];
}, app);
value =
value === undefined ? '' : value === null ? '' : value;
if (header.format) {
value = header.format(value);
}
parsePreference = false;
// there aren't typically many preferences, but if there, then a object map should be created and used
const preference = preferences.find(
(preference) => preference.multiselectQuestionId === curr,
);
multiselectQuestionValue = true;
return preference;
}

// sets parsePreference to true, for the next iteration
if (curr === 'preferences') {
parsePreference = true;
}

if (acc === null || acc === undefined) {
return '';
}

// handles working with arrays, e.g. householdMember.0.firstName
if (!isNaN(Number(curr))) {
const index = Number(curr);
return acc[index];
}

return acc[curr];
}, app);
value = value === undefined ? '' : value === null ? '' : value;
if (header.format) {
value = header.format(value);
}

row += value ? `"${value.toString().replace(/"/g, `""`)}"` : '';
if (index < csvHeaders.length - 1) {
row += ',';
}
});

try {
writableStream.write(row + '\n');
} catch (e) {
console.log('writeStream write error = ', e);
writableStream.once('drain', () => {
console.log('drain buffer');
writableStream.write(row + '\n');
row += value
? `"${value.toString().replace(/"/g, `""`)}"`
: '';
if (index < csvHeaders.length - 1) {
row += ',';
}
});
row += '\n';
});
}
});
resolve(row);
}),
);
}
const resolvedArray = await Promise.all(promiseArray);
// now loop over batched row data and write them to file
resolvedArray.forEach((row) => {
try {
writableStream.write(row);
} catch (e) {
console.log('writeStream write error = ', e);
writableStream.once('drain', () => {
console.log('drain buffer');
writableStream.write(row + '\n');
});
}
});
writableStream.end();
});
});
}

async maxHouseholdMembers(applicationIds: string[]): Promise<number> {
const maxHouseholdMembersRes = await this.prisma.householdMember.groupBy({
by: ['applicationId'],
_count: {
applicationId: true,
},
where: {
OR: applicationIds.map((id) => {
return { applicationId: id };
}),
},
orderBy: {
_count: {
applicationId: 'desc',
},
},
take: 1,
});

return maxHouseholdMembersRes && maxHouseholdMembersRes.length
? maxHouseholdMembersRes[0]._count.applicationId
: 0;
}

getHouseholdCsvHeaders(maxHouseholdMembers: number): CsvHeader[] {
const headers = [];
for (let i = 0; i < maxHouseholdMembers; i++) {
Expand Down
9 changes: 4 additions & 5 deletions api/test/unit/services/application-csv-export.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,9 @@ describe('Testing application CSV export service', () => {
});

it('tests multiselectQuestionFormat with undefined question passed', () => {
expect(service.multiselectQuestionFormat(undefined)).toBe('');
expect(
service.multiselectQuestionFormat(undefined, undefined, undefined),
).toBe('');
});

it('tests multiselectQuestionFormat', () => {
Expand Down Expand Up @@ -442,13 +444,11 @@ describe('Testing application CSV export service', () => {
jurisdictions: [{ id: 'juris id' }],
} as unknown as User;

const applications = mockApplicationSet(5, new Date());
const applications = mockApplicationSet(5, new Date(), 1);
prisma.applications.findMany = jest.fn().mockReturnValue(applications);
prisma.listings.findUnique = jest.fn().mockResolvedValue({});
permissionService.canOrThrow = jest.fn().mockResolvedValue(true);

service.maxHouseholdMembers = jest.fn().mockReturnValue(1);

prisma.multiselectQuestions.findMany = jest.fn().mockReturnValue([
{
...mockMultiselectQuestion(
Expand Down Expand Up @@ -513,7 +513,6 @@ describe('Testing application CSV export service', () => {

const applications = mockApplicationSet(3, new Date());
prisma.applications.findMany = jest.fn().mockReturnValue(applications);
service.maxHouseholdMembers = jest.fn().mockReturnValue(0);
prisma.listings.findUnique = jest.fn().mockResolvedValue({});
permissionService.canOrThrow = jest.fn().mockResolvedValue(true);

Expand Down
24 changes: 21 additions & 3 deletions api/test/unit/services/application.service.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,20 @@ import { User } from '../../../src/dtos/users/user.dto';
import { permissionActions } from '../../../src/enums/permissions/permission-actions-enum';
import { GeocodingService } from '../../../src/services/geocoding.service';

export const mockApplication = (position: number, date: Date) => {
export const mockApplication = (
position: number,
date: Date,
numberOfHouseholdMembers?: number,
) => {
let householdMember = undefined;
if (numberOfHouseholdMembers) {
householdMember = [];
for (let i = 0; i < numberOfHouseholdMembers; i++) {
householdMember.push({
id: randomUUID(),
});
}
}
return {
id: randomUUID(),
appUrl: `appUrl ${position}`,
Expand Down Expand Up @@ -93,13 +106,18 @@ export const mockApplication = (position: number, date: Date) => {
},
createdAt: date,
updatedAt: date,
householdMember: householdMember,
};
};

export const mockApplicationSet = (numberToCreate: number, date: Date) => {
export const mockApplicationSet = (
numberToCreate: number,
date: Date,
numberOfHouseholdMembers?: number,
) => {
const toReturn = [];
for (let i = 0; i < numberToCreate; i++) {
toReturn.push(mockApplication(i, date));
toReturn.push(mockApplication(i, date, numberOfHouseholdMembers));
}
return toReturn;
};
Expand Down

0 comments on commit cf27fab

Please sign in to comment.