#758 Mark submitted file errors

package.json

@@ -91,6 +91,7 @@
   },
   "dependencies": {
     "@overturebio-stack/lectern-client": "1.3.0",
+    "@types/mongoose-paginate-v2": "^1.3.11",
     "adm-zip": "^0.4.16",
     "async": "^3.0.1",
     "bcrypt-nodejs": "^0.0.3",
@@ -107,14 +108,15 @@
     "express-flash": "0.0.2",
     "express-session": "^1.16.2",
     "express-validator": "^6.14.0",
-    "migrate-mongo": "^8.2.3",
     "ini": ">=1.3.6",
     "jsonwebtoken": "^8.5.1",
     "kafkajs": "^1.12.0",
     "kind-of": "^6.0.3",
     "lodash": "^4.17.21",
+    "migrate-mongo": "^8.2.3",
     "mock-http-server": "^1.4.1",
     "mongoose": "^5.13.14",
+    "mongoose-paginate-v2": "^1.6.3",
     "mongoose-sequence": "^5.2.2",
     "morgan": "^1.10.0",
     "mquery": ">=3.2.3",

src/clinical/clinical-api.ts

@@ -31,6 +31,31 @@ import { Donor } from './clinical-entities';
 import { omit } from 'lodash';
 import { DeepReadonly } from 'deep-freeze';
 
+export type ClinicalQuery = {
+  programShortName: string;
+  page: number;
+  limit: number;
+  entityTypes: string[];
+  sort?: string;
+  donorIds?: number[];
+  submitterDonorIds?: string[];
+  completionState?: {};
+};
+
+enum CompletionStates {
+  all = 'all',
+  invalid = 'invalid',
+  complete = 'complete',
+  incomplete = 'incomplete',
+}
+
+const completionFilters = {
+  invalid: { 'schemaMetadata.isValid': false },
+  complete: { 'completionStats.coreCompletionPercentage': 100 },
+  incomplete: { 'completionStats.coreCompletionPercentage': 0 },
+  all: {},
+};
+
 class ClinicalController {
   @HasFullReadAccess()
   async findDonors(req: Request, res: Response) {
@@ -61,6 +86,57 @@ class ClinicalController {
     res.send(zip.toBuffer());
   }
 
+  @HasProgramReadAccess((req: Request) => req.params.programId)
+  async getProgramClinicalEntityData(req: Request, res: Response) {
+    const programId: string = req.params.programId;
+    const sort: string = req.query.sort || 'donorId';
+    const state: CompletionStates = req.query.completionState || CompletionStates.all;
+    const entityTypes: string[] =
+      req.query.entityTypes && req.query.entityTypes.length > 0
+        ? req.query.entityTypes.split(',')
+        : [''];
+    const completionState: {} = completionFilters[state] || {};
+    const donorIds =
+      req.query.donorIds && req.query.donorIds.length > 0
+        ? { donorId: { $in: req.query.donorIds.split(',') } }
+        : '';
+
+    const submitterDonorIds =
+      req.query.submitterDonorIds && req.query.submitterDonorIds.length > 0
+        ? { submitterId: { $in: req.query.submitterDonorIds.split(',') } }
+        : '';
+
+    const query: ClinicalQuery = {
+      ...req.query,
+      sort,
+      entityTypes,
+      donorIds,
+      submitterDonorIds,
+      completionState,
+    };
+
+    if (!programId) {
+      return ControllerUtils.badRequest(res, 'Invalid programId provided');
+    }
+
+    const entityData = await service.getPaginatedClinicalData(programId, query);
+
+    res.status(200).json(entityData);
+  }
+
+  @HasProgramReadAccess((req: Request) => req.params.programId)
+  async getProgramClinicalErrors(req: Request, res: Response) {
+    const programId = req.params.programId;
+    const query = req.query.donorIds && req.query.donorIds.split(',');
+    if (!programId) {
+      return ControllerUtils.badRequest(res, 'Invalid programId provided');
+    }
+
+    const clinicalErrors = await service.getClinicalEntityMigrationErrors(programId, query);
+
+    res.status(200).json(clinicalErrors);
+  }
+
   /**
    * Fetches data for a single clinical entity type and returns the values as a TSV. This is returned in the body of the request, not as a downloadable file.
    * @param req
@@ -131,13 +207,13 @@ class ClinicalController {
 
     const coreCompletionOverride = req.body.coreCompletionOverride || {};
 
-    const upadtedDonor = await service.updateDonorStats(donorId, coreCompletionOverride);
+    const updatedDonor = await service.updateDonorStats(donorId, coreCompletionOverride);
 
-    if (!upadtedDonor) {
+    if (!updatedDonor) {
       return ControllerUtils.notFound(res, `Donor with donorId:${donorId} not found`);
     }
 
-    return res.status(200).send(upadtedDonor);
+    return res.status(200).send(updatedDonor);
   }
 
   @HasProgramReadAccess((req: Request) => req.params.programId)

src/clinical/clinical-entities.ts

@@ -113,6 +113,12 @@ export interface ClinicalInfo {
   [field: string]: string | number | boolean | string[] | number[] | boolean[] | undefined;
 }
 
+export type ClinicalEntityData = {
+  entityName: string;
+  records: ClinicalEntity[];
+  entityFields: any;
+};
+
 export type DonorMap = Readonly<{ [submitterId: string]: Donor }>;
 
 export type DonorBySubmitterIdMap = { [k: string]: DeepReadonly<Donor> };

src/clinical/clinical-service.ts

@@ -19,13 +19,20 @@
 
 import { donorDao, DONOR_DOCUMENT_FIELDS } from './donor-repo';
 import { Errors } from '../utils';
-import { Sample, Donor } from './clinical-entities';
+import { Sample, Donor, ClinicalEntityData } from './clinical-entities';
+import { ClinicalQuery } from './clinical-api';
 import { DeepReadonly } from 'deep-freeze';
 import _ from 'lodash';
 import { forceRecalcDonorCoreEntityStats } from '../submission/submission-to-clinical/stat-calculator';
+import { migrationRepo } from '../submission/migration/migration-repo';
+import {
+  DictionaryMigration,
+  DonorMigrationError,
+  DonorMigrationErrorRecord,
+} from '../submission/migration/migration-entities';
 import * as dictionaryManager from '../dictionary/manager';
 import { loggerFor } from '../logger';
-import { WorkerTasks } from './service-worker-thread/tasks';
+import { WorkerTasks, extractEntityDataFromDonors } from './service-worker-thread/tasks';
 import { runTaskInWorkerThread } from './service-worker-thread/runner';
 
 const L = loggerFor(__filename);
@@ -163,12 +170,64 @@ export const getClinicalData = async (programId: string) => {
 
   const taskToRun = WorkerTasks.ExtractDataFromDonors;
   const taskArgs = [donors, allSchemasWithFields];
-  const data = await runTaskInWorkerThread<
-    { entityName: string; records: unknown; entityFields: any }[]
-  >(taskToRun, taskArgs);
+  const data = await runTaskInWorkerThread<ClinicalEntityData[]>(taskToRun, taskArgs);
 
   const end = new Date().getTime() / 1000;
   L.debug(`getClinicalData took ${end - start}s`);
 
   return data;
 };
+
+export const getPaginatedClinicalData = async (programId: string, query: ClinicalQuery) => {
+  if (!programId) throw new Error('Missing programId!');
+  const start = new Date().getTime() / 1000;
+
+  const allSchemasWithFields = await dictionaryManager.instance().getSchemasWithFields();
+
+  const donors = await donorDao.findByPaginatedProgramId(programId, query);
+
+  const data = extractEntityDataFromDonors(donors as Donor[], allSchemasWithFields);
+
+  const end = new Date().getTime() / 1000;
+  L.debug(`getPaginatedClinicalData took ${end - start}s`);
+
+  return data;
+};
+
+interface DonorMigration extends Omit<DictionaryMigration, 'invalidDonorsErrors '> {
+  invalidDonorsErrors: DonorMigrationError[];
+}
+
+export const getClinicalEntityMigrationErrors = async (programId: string, query: string[]) => {
+  if (!programId) throw new Error('Missing programId!');
+  const start = new Date().getTime() / 1000;
+
+  const migration: DeepReadonly<
+    DonorMigration | undefined
+  > = await migrationRepo.getLatestSuccessful();
+  const clinicalErrors: any[] = [];
+
+  if (migration) {
+    const { invalidDonorsErrors }: DeepReadonly<DonorMigration> = migration;
+    invalidDonorsErrors
+      .filter(
+        donor =>
+          donor.programId.toString() === programId && query.includes(donor.donorId.toString()),
+      )
+      .forEach(donor => {
+        const { donorId, submitterDonorId } = donor;
+        // Overwrite donor.errors + flatten entityName to simplify query
+        let errors: DonorMigrationErrorRecord[] = [];
+        donor.errors.forEach(entity => {
+          const entityName = Object.keys(entity)[0];
+          errors = errors.concat(entity[entityName].map(error => ({ ...error, entityName })));
+        });
+        clinicalErrors.push({ donorId, submitterDonorId, errors });
+      });
+  }
+
+  const end = new Date().getTime() / 1000;
+  L.debug(`getClinicalEntityMigrationErrors took ${end - start}s`);
+
+  return clinicalErrors;
+};

src/clinical/donor-repo.ts

@@ -18,9 +18,12 @@
  */
 
 import { Donor } from './clinical-entities';
-import mongoose from 'mongoose';
+import { ClinicalQuery } from './clinical-api';
+import mongoose, { PaginateModel } from 'mongoose';
+import mongoosePaginate from 'mongoose-paginate-v2';
 import { DeepReadonly } from 'deep-freeze';
 import { F, MongooseUtils, notEmpty } from '../utils';
+
 export const SUBMITTER_ID = 'submitterId';
 export const SPECIMEN_SUBMITTER_ID = 'specimen.submitterId';
 export const SPECIMEN_SAMPLE_SUBMITTER_ID = 'specimen.sample.submitterId';
@@ -44,11 +47,27 @@ export enum DONOR_DOCUMENT_FIELDS {
   FAMILY_HISTORY_ID = 'familyHistory.clinicalInfo.family_relative_id',
 }
 
+const DONOR_ENTITY_CORE_FIELDS = [
+  'donorId',
+  'submitterId',
+  'programId',
+  'gender',
+  'clinicalInfo',
+  'completionStats',
+  'specimens',
+];
+
 export type FindByProgramAndSubmitterFilter = DeepReadonly<{
   programId: string;
   submitterId: string;
 }>;
 
+export type FindPaginatedProgramFilter = {
+  programId: string;
+  submitterId?: { $in: '' };
+  donorId?: { $in: '' };
+};
+
 export interface DonorRepository {
   findByClinicalEntitySubmitterIdAndProgramId(
     filters: DeepReadonly<FindByProgramAndSubmitterFilter>,
@@ -63,6 +82,7 @@ export interface DonorRepository {
     projections?: Partial<Record<DONOR_DOCUMENT_FIELDS, number>>,
     omitMongoDocIds?: boolean,
   ): Promise<DeepReadonly<Donor[]>>;
+  findByPaginatedProgramId(programId: string, query: ClinicalQuery): Promise<DeepReadonly<Donor[]>>;
   deleteByProgramId(programId: string): Promise<void>;
   findByProgramAndSubmitterId(
     filters: DeepReadonly<FindByProgramAndSubmitterFilter[]>,
@@ -134,7 +154,6 @@ export const donorDao: DonorRepository = {
     if (omitMongoDocIds) {
       return findByProgramIdOmitMongoDocId(programId, projection);
     }
-
     const result = await DonorModel.find(
       {
         [DONOR_DOCUMENT_FIELDS.PROGRAM_ID]: programId,
@@ -152,6 +171,39 @@ export const donorDao: DonorRepository = {
     return F(mapped);
   },
 
+  async findByPaginatedProgramId(
+    programId: string,
+    query: ClinicalQuery,
+  ): Promise<DeepReadonly<Donor[]>> {
+    const { page, limit, sort, entityTypes, donorIds, submitterDonorIds, completionState } = query;
+
+    const projection = [...DONOR_ENTITY_CORE_FIELDS, ...entityTypes].join(' ');
+
+    const result = await DonorModel.paginate(
+      {
+        [DONOR_DOCUMENT_FIELDS.PROGRAM_ID]: programId,
+        ...donorIds,
+        ...submitterDonorIds,
+        ...completionState,
+      },
+      {
+        limit,
+        page,
+        projection,
+        sort,
+        select: '-_id',
+      },
+    );
+
+    const mapped: Donor[] = result.docs
+      .map((d: DonorDocument) => {
+        return MongooseUtils.toPojo(d) as Donor;
+      })
+      .filter(notEmpty);
+
+    return F(mapped);
+  },
+
   async findBySpecimenSubmitterIdAndProgramId(
     filter: FindByProgramAndSubmitterFilter,
   ): Promise<DeepReadonly<Donor> | undefined> {
@@ -492,6 +544,8 @@ DonorSchema.plugin(AutoIncrement, {
   start_seq: process.env.DONOR_ID_SEED || 250000,
 });
 
+DonorSchema.plugin(mongoosePaginate);
+
 SpecimenSchema.plugin(AutoIncrement, {
   inc_field: 'specimenId',
   start_seq: process.env.SPECIMEN_ID_SEED || 210000,
@@ -537,4 +591,6 @@ TreatmentSchema.plugin(AutoIncrement, {
   start_seq: 1,
 });
 
-export let DonorModel = mongoose.model<DonorDocument>('Donor', DonorSchema);
+export let DonorModel = mongoose.model<DonorDocument>('Donor', DonorSchema) as PaginateModel<
+  DonorDocument
+>;