Stress test more data & add server pagination/filtering/sorting

application/admin-client/package.json

@@ -8,7 +8,7 @@
     "@mui/icons-material": "^5.8.3",
     "@mui/lab": "^5.0.0-alpha.85",
     "@mui/material": "^6.1.5",
-    "@mui/x-data-grid": "^8.0.0",
+    "@mui/x-data-grid": "latest-v7",
     "@refinedev/cli": "^2.16.46",
     "@refinedev/core": "^4.57.9",
     "@refinedev/devtools": "^1.2.16",
@@ -28,6 +28,7 @@
     "react-markdown": "^10.0.1",
     "react-router-dom": "^6.8.1",
     "react-router-hash-link": "^2.4.3",
+    "x-data-grid-8": "npm:@mui/x-data-grid@8.14",
     "zustand": "^5.0.0"
   },
   "devDependencies": {

application/admin-client/src/pages/participants/list.tsx

@@ -9,12 +9,13 @@ import {
   Menu,
   MenuItem,
   Modal,
+  TextField,
   Tooltip,
   Typography,
 } from '@mui/material'
-import { DataGrid, type GridColDef } from '@mui/x-data-grid'
+import { DataGrid, GridFilterOperator, type GridColDef } from '@mui/x-data-grid'
 import { DateField, EditButton, List, ShowButton, useDataGrid } from '@refinedev/mui'
-import React, { useEffect, useState } from 'react'
+import React, { useEffect, useRef, useState } from 'react'
 import { Link, useLocation } from 'react-router-dom'
 import { InviteModal } from '../../components/InviteModal'
 import { axiosInstance } from '../../providers/dataProvider'
@@ -40,15 +41,14 @@ export const statusMap = {
 
 export const ParticipantList = () => {
   const { dataGridProps } = useDataGrid({
-    syncWithLocation: true,
-    filters: { mode: 'off' },
-    sorters: { mode: 'off' },
+    syncWithLocation: false,
+    pagination: { pageSize: 10, mode: 'server' } as any,
+    filters: { mode: 'server' },
+    sorters: { mode: 'server' },
   })
   const { dataGridProps: inviteGridProps } = useDataGrid({
-    syncWithLocation: true,
+    syncWithLocation: false,
     resource: 'invites',
-    filters: { mode: 'off' },
-    sorters: { mode: 'off' },
   })
 
   const location = useLocation()
@@ -158,30 +158,91 @@ export const ParticipantList = () => {
     )
   }
 
+  // Debounced input component
+  function DebouncedInput(props: any) {
+    const { item, applyValue, InputProps } = props
+    const [value, setValue] = useState(item.value ?? '')
+    const debounceRef = useRef<number | undefined>()
+
+    const handleChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+      const newValue = event.target.value
+      setValue(newValue)
+      if (debounceRef.current) {
+        window.clearTimeout(debounceRef.current)
+      }
+      debounceRef.current = window.setTimeout(() => {
+        applyValue({ ...item, value: newValue })
+      }, 500) // 500ms debounce
+    }
+
+    useEffect(() => {
+      return () => {
+        if (debounceRef.current) {
+          window.clearTimeout(debounceRef.current)
+        }
+      }
+    }, [])
+
+    return (
+      <div style={{ display: 'flex', alignItems: 'flex-end', height: '100%' }}>
+        <TextField
+          variant="standard"
+          value={value}
+          onChange={handleChange}
+          fullWidth
+          {...InputProps}
+        />
+      </div>
+    )
+  }
+
+  const allowedOperators: GridFilterOperator[] = [
+    {
+      label: 'Equals',
+      value: 'equals',
+      requiresFilterValue: true,
+      getApplyFilterFn: (filterItem) => (value) => value === filterItem.value,
+      InputComponent: DebouncedInput,
+    },
+    {
+      label: 'Does not equal',
+      value: 'doesNotEqual',
+      getApplyFilterFn: (filterItem) => (value) => value !== filterItem.value,
+      InputComponent: DebouncedInput,
+    },
+  ]
+
   const columns = React.useMemo<GridColDef[]>(
     () => [
       {
         field: 'participantId',
         flex: 1,
         headerName: 'ID',
+        filterOperators: allowedOperators,
       },
       {
         field: 'firstName',
         flex: 1,
         headerName: 'First Name',
         minWidth: 100,
+        sortable: false,
+        filterOperators: allowedOperators,
       },
       {
         field: 'lastName',
         flex: 1,
         headerName: 'Last Name',
         minWidth: 100,
+        sortable: false,
+        filterOperators: allowedOperators,
       },
       {
         field: 'email',
         flex: 1,
         headerName: 'Email',
         minWidth: 100,
+        sortable: false,
+        filterOperators: allowedOperators,
       },
       {
         field: 'answers',
@@ -197,6 +258,8 @@ export const ParticipantList = () => {
         headerName: 'Latest Survey Response',
         minWidth: 100,
         type: 'date',
+        disableColumnMenu: true,
+        sortable: false,
         valueGetter: (value) => {
           if (!value) return null
           return new Date(value)

application/admin-client/src/pages/responses/all.tsx

@@ -19,7 +19,7 @@ import {
   useGridSelector,
   gridFilterActiveItemsSelector,
   gridColumnVisibilityModelSelector,
-} from '@mui/x-data-grid'
+} from 'x-data-grid-8'
 import { useMemo, useState } from 'react'
 import { List } from '@refinedev/mui'
 import { styled } from '@mui/material/styles'

application/admin-client/src/providers/dataProvider.ts

@@ -33,6 +33,7 @@ axiosInstance.interceptors.response.use(
   (response) => response,
   (error) => {
     if (error.response && error.response.status === 401) {
+      localStorage.removeItem(TOKEN_KEY)
       window.location.href = '/login'
     }
     error.message = error?.response?.data?.details || error.message
@@ -64,16 +65,34 @@ export const dataProvider = (): DataProvider => {
         data,
       }
     },
-    getList: async ({ resource }) => {
+    getList: async ({ resource, pagination, filters, sorters }: any) => {
       const { studies, activeStudyIndex } = useStudyStore.getState()
       const studyId = studies[activeStudyIndex].id
       let url = resource
-      if (studyResources.includes(resource)) url = `/studies/${studyId}/${url}`
+      const params = new URLSearchParams()
+      if (pagination) {
+        params.append(
+          '_start',
+          String(((pagination.current || 1) - 1) * (pagination.pageSize || 1)),
+        )
+        params.append('_end', String((pagination.current || 1) * (pagination.pageSize || 1)))
+      }
+
+      if (filters?.at(0)) {
+        params.append(`filter[${filters[0].field}][${filters[0].operator}]`, filters[0].value)
+      }
+
+      if (sorters?.at(0)) {
+        params.append(`orderBy[${sorters[0].field}]`, sorters[0].order)
+      }
+
+      if (studyResources.includes(resource)) url = `/studies/${studyId}/${url}?${params.toString()}`
       const response = await axiosInstance.get(url)
       const data = response.data.data
+
       return {
         data,
-        total: data.length,
+        total: response.data.total || data.length,
       }
     },
     create: async ({ resource, variables }) => {

application/backend/package.json

@@ -14,6 +14,7 @@
     "prisma:seed": "yarn prisma db seed",
     "prisma:test": "yarn ts-node -T scripts/runTestSeed.ts",
     "prisma:wipe": "yarn ts-node -T scripts/runWipeDB.ts",
+    "prisma:stresstest": "yarn prisma:wipe && yarn prisma:seed && vite-node scripts/stressTestSeed.ts",
     "prisma:migrate": "yarn prisma migrate dev",
     "console": "ts-node -T -r ./prisma/console.ts"
   },
@@ -39,7 +40,8 @@
     "sharp": "^0.34.1",
     "swagger-ui-express": "^5.0.1",
     "tsoa": "^6.6.0",
-    "uuid": "^11.1.0"
+    "uuid": "^11.1.0",
+    "vite-node": "^3.2.4"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",

application/backend/scripts/stressTestSeed.ts

@@ -0,0 +1,50 @@
+import { ParticipantProfile, StudyParticipant, SurveyVersionAnswers } from '@prisma/client'
+import { createDefaultAnswers } from '../src/utils/answers'
+import prisma from '../src/PrismaClient'
+import { faker } from '@faker-js/faker'
+
+const main = async () => {
+  // Get number of records from command line argument, fallback to 1000
+  const numRecords = process.argv[2] ? parseInt(process.argv[2], 10) : 1000
+  if (isNaN(numRecords) || numRecords <= 0) {
+    console.error('Please provide a valid positive integer for the number of records.')
+    process.exit(1)
+  }
+
+  //eslint-disable-next-line
+  const SeedSurveyStepData = require('../prisma/seed/seedSurveyStepData.json')
+  const exampleAnswers = createDefaultAnswers(SeedSurveyStepData)
+
+  const participants: Partial<StudyParticipant>[] = []
+  const profiles: Partial<ParticipantProfile>[] = []
+  const answers: Partial<SurveyVersionAnswers>[] = []
+  for (let i = 0; i < numRecords; i++) {
+    const pData: Partial<ParticipantProfile> = {
+      id: 1000 + i,
+      addressLine: faker.string.alphanumeric(),
+      dob: faker.date.birthdate(),
+      firstName: faker.person.firstName(),
+      lastName: faker.person.lastName(),
+      mobile: faker.phone.number(),
+      participantType: 'STANDARD',
+      postcode: '1234',
+      preferredContact: 'EMAIL',
+      state: 'ACT',
+      suburb: 'SUBURB',
+    }
+    profiles.push(pData)
+    participants.push({
+      participantId: faker.string.uuid(),
+      participantProfileId: 1000 + i,
+      studyId: 1,
+      participantNumber: 1000 + i,
+    })
+
+    answers.push({ profileId: 1000 + i, versionId: 1000, answers: exampleAnswers })
+  }
+  await prisma.participantProfile.createMany({ data: profiles as any })
+  await prisma.studyParticipant.createMany({ data: participants as any })
+  await prisma.surveyVersionAnswers.createMany({ data: answers as any })
+}
+
+main()

application/backend/src/controllers/ParticipantsController.test.ts

@@ -61,6 +61,22 @@ describe('ParticipantsController', () => {
       )
       expect(body.data[1].answers).toHaveLength(1)
       expect(body.data[1].answers[0].status).toBe('complete')
+
+      const res2 = await request(app)
+        .get('/studies/1/participants?_start=0&_end=2')
+        .set({ Authorization: `Bearer ${organisationAdminToken}` })
+      expect(res2.body.data).toHaveLength(2)
+
+      const res3 = await request(app)
+        .get('/studies/1/participants?_start=2&_end=4')
+        .set({ Authorization: `Bearer ${organisationAdminToken}` })
+      expect([...res2.body.data, ...res3.body.data]).toEqual(body.data)
+    })
+    it('Can filter', async () => {
+      const res = await request(app)
+        .get('/studies/1/participants?_start=0&_end=10&filter[lastName][eq]=User')
+        .set({ Authorization: `Bearer ${organisationAdminToken}` })
+      expect(res.body.data).toHaveLength(2)
     })
   })