feat: 3291/listing export take 2 (#3424)

* fix: functional frontend

* fix: hooks and service updates

* fix: hitting service file

* fix: wip config work

* fix: wip config 2

* fix: completed query updates

* fix: complete column naming and basic formatting

* fix: clean up formatting

* fix: wip testing debugging

* fix: functional unit tests

* fix: cypress tests + formatting

* fix: unadded test changes

* fix: internal csv testing

* fix: exporter test fix

* fix: more detailed csv checks

* fix: testing + formatting tweaks

* fix: exporter testing improvements

* fix: updates per pr feedback

* fix: match config pattern

* fix: add close status option

* fix: reset netlify testing

* fix: final cleanup

* fix: rent type formatting

* fix: remove console log

backend/core/package.json

@@ -67,6 +67,7 @@
     "handlebars": "^4.7.6",
     "joi": "^17.3.0",
     "jwt-simple": "^0.5.6",
+    "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "nanoid": "^3.1.12",
     "nestjs-twilio": "^2.1.0",

backend/core/src/listings/dto/listings-zip-query-params.ts

@@ -0,0 +1,15 @@
+import { Expose } from "class-transformer"
+import { ApiProperty } from "@nestjs/swagger"
+import { IsOptional, IsString } from "class-validator"
+import { ValidationsGroupsEnum } from "../../shared/types/validations-groups-enum"
+
+export class ListingsZipQueryParams {
+  @Expose()
+  @ApiProperty({
+    type: String,
+    required: false,
+  })
+  @IsOptional({ groups: [ValidationsGroupsEnum.default] })
+  @IsString({ groups: [ValidationsGroupsEnum.default] })
+  timeZone?: string
+}

backend/core/src/listings/helpers.ts

@@ -0,0 +1,87 @@
+import { PaperApplication } from "../../src/paper-applications/entities/paper-application.entity"
+import { isEmpty } from "../shared/utils/is-empty"
+import { formatLocalDate } from "../shared/utils/format-local-date"
+
+export const cloudinaryPdfFromId = (publicId: string): string => {
+  if (isEmpty(publicId)) return ""
+  const cloudName = process.env.cloudinaryCloudName || process.env.CLOUDINARY_CLOUD_NAME
+  return `https://res.cloudinary.com/${cloudName}/image/upload/${publicId}.pdf`
+}
+
+export const getPaperAppUrls = (paperApps: PaperApplication[]) => {
+  if (isEmpty(paperApps)) return ""
+  const urlArr = paperApps.map((paperApplication) =>
+    cloudinaryPdfFromId(paperApplication.file?.fileId)
+  )
+  const formattedResults = urlArr.join(", ")
+  return formattedResults
+}
+
+export const formatYesNo = (value: boolean | null): string => {
+  if (value === null || typeof value == "undefined") return ""
+  else if (value) return "Yes"
+  else return "No"
+}
+
+export const formatStatus = {
+  active: "Public",
+  closed: "Closed",
+  pending: "Draft",
+}
+
+export const formatUnitType = {
+  SRO: "SRO",
+  studio: "Studio",
+  oneBdrm: "1 BR",
+  twoBdrm: "2 BR",
+  threeBdrm: "3 BR",
+  fourBdrm: "4 BR",
+  fiveBdrm: "5 BR",
+}
+
+export const formatCommunityType = {
+  senior55: "Seniors 55+",
+  senior62: "Seniors 62+",
+  specialNeeds: "Special Needs",
+}
+
+export const formatCurrency = (value: string): string => {
+  return value ? `$${value}` : ""
+}
+
+export const convertToTitleCase = (value: string): string => {
+  if (isEmpty(value)) return ""
+  const spacedValue = value.replace(/([A-Z])/g, (match) => ` ${match}`)
+  const result = spacedValue.charAt(0).toUpperCase() + spacedValue.slice(1)
+  return result
+}
+
+export const formatOpenHouse = (openHouseArr: any[], tz: string): string => {
+  const openHouseFormatted = []
+  openHouseArr.forEach((openHouse) => {
+    let openHouseStr = ""
+    if (openHouse.label) openHouseStr += `${openHouse.label}`
+    if (openHouse.startTime) {
+      const date = formatLocalDate(openHouse.startTime, "MM-DD-YYYY", tz)
+      openHouseStr += `: ${date}`
+      if (openHouse.endTime) {
+        const startTime = formatLocalDate(openHouse.startTime, "hh:mmA", tz)
+        const endTime = formatLocalDate(openHouse.endTime, "hh:mmA z", tz)
+        openHouseStr += ` (${startTime} - ${endTime})`
+      }
+    }
+    if (openHouseStr !== "") openHouseFormatted.push(openHouseStr)
+  })
+  return openHouseFormatted.join(", ")
+}
+
+export const hideZero = (fieldValue: number | string) => {
+  if (isEmpty(fieldValue) || fieldValue === 0 || fieldValue === "0") return ""
+  return fieldValue
+}
+
+export const formatRentType = (monthlyRentAsPercentOfIncome: string, monthlyRent: string) => {
+  if (!isEmpty(monthlyRentAsPercentOfIncome)) return "% of income"
+  if (!isEmpty(monthlyRent)) return "Fixed amount"
+  return ""
+}

backend/core/src/listings/listings-csv-exporter.service.ts

@@ -0,0 +1,189 @@
+import { Injectable, Scope } from "@nestjs/common"
+import { CsvBuilder } from "../applications/services/csv-builder.service"
+import {
+  cloudinaryPdfFromId,
+  formatCurrency,
+  formatStatus,
+  formatYesNo,
+  convertToTitleCase,
+  getPaperAppUrls,
+  formatUnitType,
+  formatCommunityType,
+  formatOpenHouse,
+  hideZero,
+  formatRentType,
+} from "./helpers"
+import { formatLocalDate } from "../shared/utils/format-local-date"
+import { ListingReviewOrder } from "./types/listing-review-order-enum"
+@Injectable({ scope: Scope.REQUEST })
+export class ListingsCsvExporterService {
+  constructor(private readonly csvBuilder: CsvBuilder) {}
+
+  exportListingsFromObject(listings: any[], users: any[], timeZone: string): string {
+    // restructure user information to listingId->user rather than user->listingId
+    const partnerAccessHelper = {}
+    users.forEach((user) => {
+      const userName = `${user.firstName} ${user.lastName}`
+      user.leasingAgentInListings.forEach((listing) => {
+        partnerAccessHelper[listing.id]
+          ? partnerAccessHelper[listing.id].push(userName)
+          : (partnerAccessHelper[listing.id] = [userName])
+      })
+    })
+    const listingObj = listings.map((listing) => {
+      const programsFormatted = []
+      const preferencesFormatted = []
+      listing.listingMultiselectQuestions?.forEach((question) => {
+        const questionInfo = question.multiselectQuestion
+        if (questionInfo?.applicationSection === "preferences")
+          preferencesFormatted.push(questionInfo?.text)
+        else if (questionInfo?.applicationSection === "programs")
+          programsFormatted.push(questionInfo?.text)
+      })
+
+      const openHouse = []
+      let lottery
+      listing.events?.forEach((event) => {
+        if (event.type === "openHouse") {
+          openHouse.push(event)
+        } else if (event.type === "publicLottery") {
+          lottery = event
+        }
+      })
+
+      return {
+        ID: listing.id,
+        "Created At Date": formatLocalDate(listing.createdAt, "MM-DD-YYYY hh:mm:ssA z", timeZone),
+        Jurisdiction: listing.jurisdiction.name,
+        "Listing Name": listing.name,
+        "Listing Status": formatStatus[listing.status],
+        "Publish Date": formatLocalDate(listing.publishedAt, "MM-DD-YYYY hh:mm:ssA z", timeZone),
+        "Last Updated": formatLocalDate(listing.updatedAt, "MM-DD-YYYY hh:mm:ssA z", timeZone),
+        Developer: listing.developer,
+        "Building Street Address": listing.buildingAddress?.street,
+        "Building City": listing.buildingAddress?.city,
+        "Building State": listing.buildingAddress?.state,
+        "Building Zip": listing.buildingAddress?.zipCode,
+        "Building Neighborhood": listing?.neighborhood,
+        "Building Year Built": listing.yearBuilt,
+        "Reserved Community Types": formatCommunityType[listing.reservedCommunityType?.name],
+        Latitude: listing.buildingAddress?.latitude,
+        Longitude: listing.buildingAddress?.longitude,
+        "Number of units": listing.numberOfUnits,
+        "Listing Availability":
+          listing?.reviewOrderType === ListingReviewOrder.waitlist
+            ? "Open Waitlist"
+            : "Available Units",
+        "Important Program Rules": listing.programRules,
+        "Review Order": convertToTitleCase(listing.reviewOrderType),
+        "Lottery Date": formatLocalDate(lottery?.startTime, "MM-DD-YYYY", timeZone),
+        "Lottery Start": formatLocalDate(lottery?.startTime, "hh:mmA z", timeZone),
+        "Lottery End": formatLocalDate(lottery?.endTime, "hh:mmA z", timeZone),
+        "Lottery Notes": listing.events[0]?.note,
+        "Housing Preferences": preferencesFormatted.join(", "),
+        "Housing Programs": programsFormatted.join(", "),
+        "Application Fee": formatCurrency(listing.applicationFee),
+        "Deposit Helper Text": listing.depositHelperText,
+        "Deposit Min": formatCurrency(listing.depositMin),
+        "Deposit Max": formatCurrency(listing.depositMax),
+        "Costs Not Included": listing.costsNotIncluded,
+        "Property Amenities": listing.amenities,
+        "Additional Accessibility": listing.accessibility,
+        "Unit Amenities": listing.unitAmenities,
+        "Smoking Policy": listing.smokingPolicy,
+        "Pets Policy": listing.petPolicy,
+        "Services Offered": listing.servicesOffered,
+        "Eligibility Rules - Credit History": listing.creditHistory,
+        "Eligibility Rules - Rental History": listing.rentalHistory,
+        "Eligibility Rules - Criminal Background": listing.criminalBackground,
+        "Eligibility Rules - Rental Assistance": listing.rentalAssistance,
+        "Building Selection Criteria":
+          listing.buildingSelectionCriteria ??
+          cloudinaryPdfFromId(listing.buildingSelectionCriteriaFile?.fileId),
+        "Required Documents": listing.requiredDocuments,
+        "Special Notes": listing.specialNotes,
+        Waitlist: formatYesNo(listing.isWaitlistOpen),
+        "Max Waitlist Size": listing.waitlistMaxSize,
+        "Leasing Agent Name": listing.leasingAgentName,
+        "Leasing Agent Email": listing.leasingAgentEmail,
+        "Leasing Agent Phone": listing.leasingAgentPhone,
+        "Leasing Agent Title": listing.leasingAgentTitle,
+        "Leasing Agent Office Hours": listing.leasingAgentOfficeHours,
+        "Leasing Agent Street Address": listing.leasingAgentAddress?.street,
+        "Leasing Agent Apt/Unit #": listing.leasingAgentAddress?.street2,
+        "Leasing Agent City": listing.leasingAgentAddress?.city,
+        "Leasing Agent Zip": listing.leasingAgentAddress?.zipCode,
+        "Leasing Agency Mailing Address": listing.applicationMailingAddress?.street,
+        "Leasing Agency Mailing Address Street 2": listing.applicationMailingAddress?.street2,
+        "Leasing Agency Mailing Address City": listing.applicationMailingAddress?.city,
+        "Leasing Agency Mailing Address Zip": listing.applicationMailingAddress?.zipCode,
+        "Leasing Agency Pickup Address": listing.applicationPickUpAddress?.street,
+        "Leasing Agency Pickup Address Street 2": listing.applicationPickUpAddress?.street2,
+        "Leasing Agency Pickup Address City": listing.applicationPickUpAddress?.city,
+        "Leasing Agency Pickup Address Zip": listing.applicationPickUpAddress?.zipCode,
+        "Leasing Pick Up Office Hours": listing.applicationPickUpAddressOfficeHours,
+        "Digital Application": formatYesNo(listing.digitalApplication),
+        "Digital Application URL": listing.applicationMethods[1]?.externalReference,
+        "Paper Application": formatYesNo(listing.paperApplication),
+        "Paper Application URL": getPaperAppUrls(listing.applicationMethods[0]?.paperApplications),
+        "Referral opportunity?": formatYesNo(listing.referralOpportunity),
+        "Can applications be mailed in?": formatYesNo(
+          listing.applicationMailingAddress || listing.applicationMailingAddressType
+        ),
+        "Can applications be picked up?": formatYesNo(
+          listing.applicationPickUpAddress || listing?.applicationPickUpAddressType
+        ),
+        "Can applications be dropped off?": formatYesNo(
+          listing.applicationPickUpAddress || listing?.applicationPickUpAddressType
+        ),
+        Postmark: formatLocalDate(
+          listing.postmarkedApplicationsReceivedByDate,
+          "MM-DD-YYYY hh:mm:ssA z",
+          timeZone
+        ),
+        "Additional Application Submission Notes": listing.additionalApplicationSubmissionNotes,
+        "Application Due Date": formatLocalDate(listing.applicationDueDate, "MM-DD-YYYY", timeZone),
+        "Application Due Time": formatLocalDate(listing.applicationDueDate, "hh:mmA z", timeZone),
+        "Open House": formatOpenHouse(openHouse, timeZone),
+        "Partners Who Have Access": partnerAccessHelper[listing.id]?.join(", "),
+      }
+    })
+    return this.csvBuilder.buildFromIdIndex(listingObj)
+  }
+
+  exportUnitsFromObject(listings: any[]): string {
+    const reformattedListings = []
+    listings.forEach((listing) => {
+      listing.units?.forEach((unit) => {
+        reformattedListings.push({
+          id: listing.id,
+          name: listing.name,
+          unit,
+        })
+      })
+    })
+    const unitsFormatted = reformattedListings.map((listing) => {
+      return {
+        "Listing ID": listing.id,
+        "Listing Name": listing.name,
+        "Unit Number": listing.unit?.number,
+        "Unit Type": formatUnitType[listing.unit?.unitType?.name],
+        "Number of Bathrooms": listing.unit?.numBathrooms,
+        "Unit Floor": hideZero(listing.unit?.floor),
+        "Square footage": listing.unit?.sqFeet,
+        "Minimum Occupancy": hideZero(listing.unit?.minOccupancy),
+        "Max Occupancy": hideZero(listing.unit?.maxOccupancy),
+        "AMI Chart": listing.unit?.amiChart?.name,
+        "AMI Level": listing.unit?.amiChart?.items[0]?.percentOfAmi,
+        "Rent Type": formatRentType(
+          listing.unit?.monthlyRentAsPercentOfIncome,
+          listing.unit?.monthlyRent
+        ),
+        "Monthly Rent": listing.unit?.monthlyRentAsPercentOfIncome ?? listing.unit?.monthlyRent,
+        "Minimum Income": listing.unit?.monthlyIncomeMin,
+        "Accessibility Priority Type": listing.unit?.priorityType?.name,
+      }
+    })
+    return this.csvBuilder.buildFromIdIndex(unitsFormatted)
+  }
+}

backend/core/src/listings/listings.controller.ts

@@ -14,6 +14,7 @@ import {
   ClassSerializerInterceptor,
   Headers,
   ParseUUIDPipe,
+  Header,
 } from "@nestjs/common"
 import { ListingsService } from "./listings.service"
 import { ApiBearerAuth, ApiExtraModels, ApiOperation, ApiTags } from "@nestjs/swagger"
@@ -35,6 +36,9 @@ import { ActivityLogInterceptor } from "../activity-log/interceptors/activity-lo
 import { ActivityLogMetadata } from "../activity-log/decorators/activity-log-metadata.decorator"
 import { ListingsApiExtraModels } from "./types/listings-api-extra-models"
 import { IdDto } from "../shared/dto/id.dto"
+import { AuthzGuard } from "../../src/auth/guards/authz.guard"
+import { ListingsCsvExporterService } from "./listings-csv-exporter.service"
+import { ListingsZipQueryParams } from "./dto/listings-zip-query-params"
 
 @Controller("listings")
 @ApiTags("listings")
@@ -45,7 +49,10 @@ import { IdDto } from "../shared/dto/id.dto"
 @ActivityLogMetadata([{ targetPropertyName: "status", propertyPath: "status" }])
 @UseInterceptors(ActivityLogInterceptor)
 export class ListingsController {
-  constructor(private readonly listingsService: ListingsService) {}
+  constructor(
+    private readonly listingsService: ListingsService,
+    private readonly listingsCsvExporter: ListingsCsvExporterService
+  ) {}
 
   // TODO: Limit requests to defined fields
   @Get()
@@ -65,6 +72,24 @@ export class ListingsController {
     return mapTo(ListingDto, listing)
   }
 
+  @Get(`csv`)
+  @UseGuards(OptionalAuthGuard, AuthzGuard)
+  @ApiOperation({ summary: "Retrieve listings and units in csv", operationId: "listAsCsv" })
+  @Header("Content-Type", "text/csv")
+  async listAsCsv(
+    @Query(new ValidationPipe(defaultValidationPipeOptions))
+    queryParams: ListingsZipQueryParams
+  ): Promise<{ listingCsv: string; unitCsv: string }> {
+    const data = await this.listingsService.rawListWithFlagged()
+    const listingCsv = this.listingsCsvExporter.exportListingsFromObject(
+      data?.listingData,
+      data?.userAccessData,
+      queryParams.timeZone
+    )
+    const unitCsv = this.listingsCsvExporter.exportUnitsFromObject(data?.unitData)
+    return { listingCsv, unitCsv }
+  }
+
   @Get(`:id`)
   @ApiOperation({ summary: "Get listing by id", operationId: "retrieve" })
   @UseInterceptors(ClassSerializerInterceptor)

backend/core/src/listings/listings.module.ts

@@ -16,6 +16,8 @@ import { ListingRepository } from "./db/listing.repository"
 import { ListingUtilities } from "./entities/listing-utilities.entity"
 import { ApplicationFlaggedSetsModule } from "../application-flagged-sets/application-flagged-sets.module"
 import { ListingsCronService } from "./listings-cron.service"
+import { ListingsCsvExporterService } from "./listings-csv-exporter.service"
+import { CsvBuilder } from "../../src/applications/services/csv-builder.service"
 
 @Module({
   imports: [
@@ -35,7 +37,7 @@ import { ListingsCronService } from "./listings-cron.service"
     ApplicationFlaggedSetsModule,
     HttpModule,
   ],
-  providers: [ListingsService, ListingsCronService, Logger],
+  providers: [ListingsService, ListingsCronService, Logger, CsvBuilder, ListingsCsvExporterService],
   exports: [ListingsService],
   controllers: [ListingsController],
 })