feat: convert from doi to pmid and create imperative pmid fetch function

compose/neurosynth-frontend/src/components/CurationComponents/CurationImport/CurationDoImport/ImportPMIDs/FetchPMIDs.tsx

@@ -11,11 +11,7 @@ const FetchPMIDs: React.FC<{
     pubmedIds: string[];
     onStubsUploaded: (stubs: ICurationStubStudy[], unimportedStubs?: string[]) => void;
 }> = React.memo((props) => {
-    const results = useGetPubmedIDs(props.pubmedIds);
-    const isLoading = results.some((x) => x.isLoading);
-    const isError = results.some((x) => x.isError);
-    const isSuccess = results.every((x) => x.isSuccess);
-    const data = results.map((x) => x.data || []);
+    const { data, isLoading, isError, isSuccess } = useGetPubmedIDs(props.pubmedIds, true);
 
     useEffect(() => {
         if (data.length === 0 || isLoading || !isSuccess) return;

compose/neurosynth-frontend/src/components/SleuthImportWizard/SleuthImportWizard.utils.ts

@@ -432,18 +432,16 @@ export const createUpdateRequestForEachSleuthStudy = (
     return Array.from(baseStudyMap).map(([_key, value]) => value);
 };
 
-export const executeHTTPRequestsAsBatches = async (
-    requestList: {
-        studyId: string;
-        data: StudyRequest;
-        request: 'UPDATE' | 'CREATE';
-    }[],
-    // called after every batch has been executed
+export const executeHTTPRequestsAsBatches = async <T, Y>(
+    requestList: T[],
+    mapFunc: (request: T) => Promise<AxiosResponse<Y>>,
+    rateLimit: number,
+    delayInMS?: number,
     callbackFunc?: (progress: number) => void
 ) => {
     const arrayOfRequestArrays = [];
-    for (let i = 0; i < requestList.length; i += 5) {
-        arrayOfRequestArrays.push(requestList.slice(i, i + 5));
+    for (let i = 0; i < requestList.length; i += rateLimit) {
+        arrayOfRequestArrays.push(requestList.slice(i, i + rateLimit));
     }
 
     const batchedResList = [];
@@ -453,24 +451,18 @@ export const executeHTTPRequestsAsBatches = async (
          * the promises are not lazy. The HTTP requests are launched as soon as
          * the function is called regardless of whether a .then() is added
          */
-        const batchedRes = await Promise.all(
-            requests.map((request) => {
-                return request.request === 'CREATE'
-                    ? API.NeurostoreServices.StudiesService.studiesPost(
-                          undefined,
-                          request.studyId,
-                          request.data
-                      )
-                    : API.NeurostoreServices.StudiesService.studiesIdPut(
-                          request.studyId,
-                          request.data
-                      );
-            })
-        );
+        const batchedRes = await Promise.all(requests.map(mapFunc));
         batchedResList.push(...batchedRes);
         if (callbackFunc) {
             callbackFunc(Math.round((batchedResList.length / requestList.length) * 100));
         }
+        if (delayInMS) {
+            await new Promise((res) => {
+                setTimeout(() => {
+                    res(null);
+                }, delayInMS);
+            });
+        }
     }
     return batchedResList;
 };

compose/neurosynth-frontend/src/components/SleuthImportWizard/SleuthImportWizardBuild.tsx

@@ -12,6 +12,7 @@ import {
     useCreateStudyset,
     useUpdateStudy,
 } from 'hooks';
+import useGetPubMedIdFromDOI, { IESearchResult } from 'hooks/external/useGetPubMedIdFromDOI';
 import { ICurationMetadata, IProvenance } from 'hooks/projects/useGetProjects';
 import useIngest from 'hooks/studies/useIngest';
 import {
@@ -26,6 +27,7 @@ import {
     initCurationHelper,
 } from 'pages/Projects/ProjectPage/ProjectStore.helpers';
 import { useEffect, useRef, useState } from 'react';
+import API from 'utils/api';
 import {
     ISleuthFileUploadStubs,
     createUpdateRequestForEachSleuthStudy,
@@ -34,12 +36,15 @@ import {
     sleuthStubsToBaseStudies,
     updateUploadSummary,
 } from './SleuthImportWizard.utils';
+import useGetPubmedIDs from 'hooks/external/useGetPubMedIds';
 
 const SleuthImportWizardBuild: React.FC<{
     sleuthUploads: ISleuthFileUploadStubs[];
     onNext: (projectId: string, studysetId: string, annotationId: string) => void;
 }> = (props) => {
     const { user } = useAuth0();
+    const { queryImperatively } = useGetPubmedIDs([], false);
+    const { mutateAsync: getPubMedIdFromDOI } = useGetPubMedIdFromDOI();
     const [progressValue, setProgressValue] = useState(0);
     const [progressText, setProgressText] = useState('');
     const { mutateAsync: ingestAsync } = useIngest();
@@ -199,14 +204,31 @@ const SleuthImportWizardBuild: React.FC<{
                     user!.sub as string
                 );
                 const percentageIncrement = 25 / sleuthUploads.length;
-                const percentageAlreadyComplete = percentageIncrement * index;
-                const responses = await executeHTTPRequestsAsBatches(requestList, (progress) => {
-                    setProgressValue(
-                        Math.round(
-                            (progress / 100) * percentageIncrement + percentageAlreadyComplete
-                        )
-                    );
-                });
+                const percentageAlreadyComplete = 50 + percentageIncrement * index;
+                const responses = await executeHTTPRequestsAsBatches(
+                    requestList,
+                    (requestObject) => {
+                        return requestObject.request === 'CREATE'
+                            ? API.NeurostoreServices.StudiesService.studiesPost(
+                                  undefined,
+                                  requestObject.studyId,
+                                  requestObject.data
+                              )
+                            : API.NeurostoreServices.StudiesService.studiesIdPut(
+                                  requestObject.studyId,
+                                  requestObject.data
+                              );
+                    },
+                    5,
+                    undefined,
+                    (progress) => {
+                        setProgressValue(
+                            Math.round(
+                                (progress / 100) * percentageIncrement + percentageAlreadyComplete
+                            )
+                        );
+                    }
+                );
                 databaseResponses.push({
                     responses: responses,
                     fileName: sleuthUpload.fileName,
@@ -216,49 +238,101 @@ const SleuthImportWizardBuild: React.FC<{
             return databaseResponses;
         };
 
+        const hydrateStudiesWithStudyDetails = async (baseStudies: BaseStudy[]) => {
+            const responses = await executeHTTPRequestsAsBatches(
+                baseStudies,
+                (baseStudy) => {
+                    if (baseStudy.pmid) {
+                        return new Promise<AxiosResponse<IESearchResult>>((res) => {
+                            const fakeAxiosResponse: AxiosResponse = {
+                                data: {
+                                    esearchresult: {
+                                        count: '1',
+                                        idlist: [baseStudy.pmid],
+                                    },
+                                    header: {
+                                        type: 'NEUROSTORE_MOCK',
+                                        version: 'NA',
+                                    },
+                                },
+                                status: 200,
+                                statusText: 'OK',
+                                headers: {},
+                                config: {},
+                            };
+                            res(fakeAxiosResponse);
+                        });
+                    } else {
+                        // we know that if a study does not have a PMID, then it must at least have a DOI because
+                        // we have already validated the uploaded files
+                        return getPubMedIdFromDOI(baseStudy.doi as string);
+                    }
+                },
+                3,
+                1200,
+                (progress) => {
+                    setProgressValue(Math.round((progress / 100) * 50));
+                }
+            );
+
+            const pubmedIds = responses.map((response) => {
+                return response.data.esearchresult.idlist[0];
+            });
+
+            const pubmedStudyDetails = await queryImperatively(pubmedIds);
+            console.log({ pubmedStudyDetails });
+        };
+
         const build = async (sleuthUploads: ISleuthFileUploadStubs[]) => {
             if (!user?.sub) return;
             const baseStudies = sleuthStubsToBaseStudies(sleuthUploads);
             try {
                 setProgressValue(0);
-                setProgressText('Ingesting...');
+                setProgressText('Fetching study details...');
 
-                const ingestRes = await ingest(baseStudies, sleuthUploads);
-                const responsesToIds = ingestRes.reduce((acc, curr) => {
-                    return [...acc, ...curr.responses.map((x) => x.data.id as string)];
-                }, [] as string[]);
-                setProgressValue(25);
-                setProgressText('Creating studyset...');
-
-                const createdStudyset = await handleCreateStudyset(responsesToIds);
-                if (!createdStudyset.data.id) throw new Error('Created studyset but found no ID');
-                setProgressValue(50);
-                setProgressText('Creating annotation...');
-
-                const createdAnnotation = await handleCreateAnnotation(
-                    createdStudyset.data.id as string,
-                    ingestRes
-                );
-                setProgressValue(75);
-                setProgressText('Finalizing project...');
-
-                if (!createdAnnotation.data.id)
-                    throw new Error('Created annotation but found no ID');
-                const createdProject = await handleCreateProject(
-                    createdStudyset.data.id,
-                    createdAnnotation.data.id,
-                    ingestRes
-                );
-                if (!createdProject.data.id) throw new Error('Created project but found no ID');
+                const hydratedBaseStudies = await hydrateStudiesWithStudyDetails(baseStudies);
 
-                setCreatedProjectComponents({
-                    projectId: createdProject.data.id,
-                    studysetId: createdStudyset.data.id,
-                    annotationId: createdAnnotation.data.id,
-                });
-                setProgressValue(100);
-                setProgressText('Complete...');
-                setIsLoadingState(false);
+                return;
+
+                // setProgressValue(50);
+                // setProgressText('Ingesting...');
+
+                // const ingestRes = await ingest(baseStudies, sleuthUploads);
+                // const responsesToIds = ingestRes.reduce((acc, curr) => {
+                //     return [...acc, ...curr.responses.map((x) => x.data.id as string)];
+                // }, [] as string[]);
+                // setProgressValue(75);
+                // setProgressText('Creating studyset...');
+
+                // const createdStudyset = await handleCreateStudyset(responsesToIds);
+                // if (!createdStudyset.data.id) throw new Error('Created studyset but found no ID');
+                // setProgressValue(80);
+                // setProgressText('Creating annotation...');
+
+                // const createdAnnotation = await handleCreateAnnotation(
+                //     createdStudyset.data.id as string,
+                //     ingestRes
+                // );
+                // setProgressValue(90);
+                // setProgressText('Finalizing project...');
+
+                // if (!createdAnnotation.data.id)
+                //     throw new Error('Created annotation but found no ID');
+                // const createdProject = await handleCreateProject(
+                //     createdStudyset.data.id,
+                //     createdAnnotation.data.id,
+                //     ingestRes
+                // );
+                // if (!createdProject.data.id) throw new Error('Created project but found no ID');
+
+                // setCreatedProjectComponents({
+                //     projectId: createdProject.data.id,
+                //     studysetId: createdStudyset.data.id,
+                //     annotationId: createdAnnotation.data.id,
+                // });
+                // setProgressValue(100);
+                // setProgressText('Complete...');
+                // setIsLoadingState(false);
             } catch (e) {
                 setIsError(true);
             }
@@ -275,6 +349,8 @@ const SleuthImportWizardBuild: React.FC<{
         createAnnotation,
         sleuthUploads,
         user,
+        queryImperatively,
+        getPubMedIdFromDOI,
     ]);
 
     const handleNext = () => {

compose/neurosynth-frontend/src/hooks/external/useGetPubMedIdFromDOI.tsx

@@ -0,0 +1,35 @@
+import axios, { AxiosError, AxiosResponse } from 'axios';
+import { useMutation } from 'react-query';
+
+/**
+ * NOTE: this is a get request but we use useMutation so that we can query the data imperatively.
+ * This means that there is no smart refetching
+ * https://github.com/TanStack/query/discussions/3675
+ */
+
+export interface IESearchResult {
+    esearchresult: {
+        count: string;
+        idlist: string[];
+    };
+    header: {
+        type: string;
+        version: string;
+    };
+    status: string;
+    responseDate: string;
+}
+
+const ESEARCH_UR = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi';
+
+const useGetPubMedIdFromDOI = () => {
+    return useMutation<AxiosResponse<IESearchResult>, AxiosError, string, unknown>((doi) =>
+        axios.get<IESearchResult>(`${ESEARCH_UR}?db=pubmed&retmode=json&term=${doi}`, {
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+            },
+        })
+    );
+};
+
+export default useGetPubMedIdFromDOI;