From 718397f069d8388ccf7b9d835fd0b14770ecd50f Mon Sep 17 00:00:00 2001 From: Giorgio Sironi Date: Mon, 13 May 2024 10:50:10 +0100 Subject: [PATCH] Make ingestDays available to evaluation discoverers #3234 --- ...cover-evaluations-for-access-microbiology-via-crossref.ts | 4 +++- .../discover-evaluations-from-crossref-via-biorxiv.ts | 2 +- .../discover-evaluations-from-hypothesis-group.ts | 2 +- .../discover-evaluations-from-hypothesis-user.ts | 2 +- src/ingest/evaluation-discovery/discover-pci-evaluations.ts | 2 +- .../evaluation-discovery/discover-prereview-evaluations.ts | 2 +- .../discover-rapid-reviews-evaluations.ts | 2 +- .../third-parties/prelights/fetch-prelights-evaluations.ts | 2 +- src/ingest/update-all.ts | 5 ++++- .../evaluation-discovery/discover-pci-evaluations.test.ts | 3 ++- .../discover-prereview-evaluations.test.ts | 3 ++- .../discover-rapid-reviews-evaluations.test.ts | 3 ++- test/ingest/evaluation-discovery/ingest-days.helper.ts | 1 + 13 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 test/ingest/evaluation-discovery/ingest-days.helper.ts diff --git a/src/ingest/evaluation-discovery/discover-evaluations-for-access-microbiology-via-crossref.ts b/src/ingest/evaluation-discovery/discover-evaluations-for-access-microbiology-via-crossref.ts index d932c7996f..e771a4f47d 100644 --- a/src/ingest/evaluation-discovery/discover-evaluations-for-access-microbiology-via-crossref.ts +++ b/src/ingest/evaluation-discovery/discover-evaluations-for-access-microbiology-via-crossref.ts @@ -58,7 +58,9 @@ const toHumanFriendlyErrorMessage = ( (formattedErrors) => `acmi: could not decode crossref response ${formattedErrors.join(', ')}`, ); -export const discoverEvaluationsForAccessMicrobiologyViaCrossref: DiscoverPublishedEvaluations = (dependencies) => pipe( +export const discoverEvaluationsForAccessMicrobiologyViaCrossref: DiscoverPublishedEvaluations = () => ( + dependencies, +) => pipe( 'https://api.crossref.org/works?filter=prefix:10.1099,type:peer-review,relation.type:is-review-of&sort=published&order=asc', dependencies.fetchData, TE.chainEitherK(flow( diff --git a/src/ingest/evaluation-discovery/discover-evaluations-from-crossref-via-biorxiv.ts b/src/ingest/evaluation-discovery/discover-evaluations-from-crossref-via-biorxiv.ts index e576cd24a9..585d18cb15 100644 --- a/src/ingest/evaluation-discovery/discover-evaluations-from-crossref-via-biorxiv.ts +++ b/src/ingest/evaluation-discovery/discover-evaluations-from-crossref-via-biorxiv.ts @@ -71,7 +71,7 @@ const identifyCandidates = (doiPrefix: string, reviewDoiPrefix: string) => { export const discoverEvaluationsFromCrossrefViaBiorxiv = ( doiPrefix: string, reviewDoiPrefix: string, -): DiscoverPublishedEvaluations => () => pipe( +): DiscoverPublishedEvaluations => () => () => pipe( identifyCandidates(doiPrefix, reviewDoiPrefix), TE.map(RA.map(toEvaluation)), TE.map((evaluations) => ({ diff --git a/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-group.ts b/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-group.ts index a8a6615edd..c2748f7d66 100644 --- a/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-group.ts +++ b/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-group.ts @@ -21,7 +21,7 @@ const calculateEarliestPublicationDateToConsider = (earliestPublicationDateToCon export const discoverEvaluationsFromHypothesisGroup = ( publisherGroupId: string, avoidWhenPublishedBefore?: Date, -): DiscoverPublishedEvaluations => (ports: Ports) => pipe( +): DiscoverPublishedEvaluations => () => (ports: Ports) => pipe( publisherGroupId, Hyp.fetchEvaluationsByGroupSince( calculateEarliestPublicationDateToConsider(avoidWhenPublishedBefore), diff --git a/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-user.ts b/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-user.ts index 8e9a19ebde..75b2972208 100644 --- a/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-user.ts +++ b/src/ingest/evaluation-discovery/discover-evaluations-from-hypothesis-user.ts @@ -15,7 +15,7 @@ type Ports = { export const discoverEvaluationsFromHypothesisUser = ( publisherUserId: string, daysToLookBackForAcceptablePerformance = 5, -): DiscoverPublishedEvaluations => (ports: Ports) => pipe( +): DiscoverPublishedEvaluations => () => (ports: Ports) => pipe( publisherUserId, Hyp.fetchEvaluationsByUserSince(ingestionWindowStartDate(daysToLookBackForAcceptablePerformance), ports.fetchData), TE.map(RA.map(convertHypothesisAnnotationToEvaluation(tagToEvaluationTypeMap))), diff --git a/src/ingest/evaluation-discovery/discover-pci-evaluations.ts b/src/ingest/evaluation-discovery/discover-pci-evaluations.ts index 476e8ada92..95dce1d9ac 100644 --- a/src/ingest/evaluation-discovery/discover-pci-evaluations.ts +++ b/src/ingest/evaluation-discovery/discover-pci-evaluations.ts @@ -75,7 +75,7 @@ type Ports = { fetchData: FetchData, }; -export const discoverPciEvaluations = (url: string): DiscoverPublishedEvaluations => (ports: Ports) => pipe( +export const discoverPciEvaluations = (url: string): DiscoverPublishedEvaluations => () => (ports: Ports) => pipe( ports.fetchData(url), TE.map(identifyCandidates), TE.map(RA.map(toEvaluationOrSkip)), diff --git a/src/ingest/evaluation-discovery/discover-prereview-evaluations.ts b/src/ingest/evaluation-discovery/discover-prereview-evaluations.ts index 4a8f4dbbdd..03811ff6ca 100644 --- a/src/ingest/evaluation-discovery/discover-prereview-evaluations.ts +++ b/src/ingest/evaluation-discovery/discover-prereview-evaluations.ts @@ -90,7 +90,7 @@ const identifyCandidates = (fetchData: FetchData) => pipe( )), ); -export const discoverPrereviewEvaluations = (): DiscoverPublishedEvaluations => (ports: Ports) => pipe( +export const discoverPrereviewEvaluations = (): DiscoverPublishedEvaluations => () => (ports: Ports) => pipe( identifyCandidates(ports.fetchData), TE.map(RA.map(toEvaluationOrSkip)), TE.map((parts) => ({ diff --git a/src/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.ts b/src/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.ts index b93f5c607a..81274d1351 100644 --- a/src/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.ts +++ b/src/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.ts @@ -36,7 +36,7 @@ const toEvaluationOrSkip = (candidate: CR.CrossrefReview) => pipe( })), ); -export const discoverRapidReviewsEvaluations = (): DiscoverPublishedEvaluations => (ports: Ports) => pipe( +export const discoverRapidReviewsEvaluations = (): DiscoverPublishedEvaluations => () => (ports: Ports) => pipe( identifyCandidates(ports.fetchData), TE.map(RA.map(toEvaluationOrSkip)), TE.map((parts) => ({ diff --git a/src/ingest/third-parties/prelights/fetch-prelights-evaluations.ts b/src/ingest/third-parties/prelights/fetch-prelights-evaluations.ts index 61a1594a38..52227fe5c5 100644 --- a/src/ingest/third-parties/prelights/fetch-prelights-evaluations.ts +++ b/src/ingest/third-parties/prelights/fetch-prelights-evaluations.ts @@ -15,7 +15,7 @@ type Dependencies = { fetchData: FetchData, }; -export const fetchPrelightsEvaluations = (): DiscoverPublishedEvaluations => (dependencies: Dependencies) => pipe( +export const fetchPrelightsEvaluations = (): DiscoverPublishedEvaluations => () => (dependencies: Dependencies) => pipe( keyFromEnv, TE.fromEither, TE.map((key) => `https://prelights.biologists.com/feed/sciety/?key=${key}&hours=120`), diff --git a/src/ingest/update-all.ts b/src/ingest/update-all.ts index a3077d8a2f..39f39dcd9a 100644 --- a/src/ingest/update-all.ts +++ b/src/ingest/update-all.ts @@ -14,6 +14,8 @@ import { DiscoveredPublishedEvaluations } from './types/discovered-published-eva type Adapters = { fetchData: FetchData }; export type DiscoverPublishedEvaluations = ( + ingestDays: number, +) => ( adapters: Adapters ) => TE.TaskEither; @@ -144,7 +146,8 @@ const sendRecordEvaluationCommands = ( ); const updateGroup = (config: Config) => (group: GroupIngestionConfiguration): TE.TaskEither => pipe( - group.discoverPublishedEvaluations({ fetchData }), + { fetchData }, + group.discoverPublishedEvaluations(config.ingestDays), TE.bimap( (error) => ({ groupName: group.name, diff --git a/test/ingest/evaluation-discovery/discover-pci-evaluations.test.ts b/test/ingest/evaluation-discovery/discover-pci-evaluations.test.ts index d75621efe4..a6e7e29e4c 100644 --- a/test/ingest/evaluation-discovery/discover-pci-evaluations.test.ts +++ b/test/ingest/evaluation-discovery/discover-pci-evaluations.test.ts @@ -1,6 +1,7 @@ import * as E from 'fp-ts/Either'; import * as TE from 'fp-ts/TaskEither'; import { pipe } from 'fp-ts/function'; +import { arbitraryIngestDays } from './ingest-days.helper'; import { discoverPciEvaluations } from '../../../src/ingest/evaluation-discovery/discover-pci-evaluations'; import { ingestionWindowStartDate } from '../../../src/ingest/time'; import { constructPublishedEvaluation } from '../../../src/ingest/types/published-evaluation'; @@ -13,7 +14,7 @@ const discover = (xml: string) => pipe( fetchData: () => TE.right(xml as unknown as D), fetchGoogleSheet: shouldNotBeCalled, }, - discoverPciEvaluations(arbitraryUri()), + discoverPciEvaluations(arbitraryUri())(arbitraryIngestDays()), ); describe('discover-pci-evaluations', () => { diff --git a/test/ingest/evaluation-discovery/discover-prereview-evaluations.test.ts b/test/ingest/evaluation-discovery/discover-prereview-evaluations.test.ts index ccb67faed3..a7365cd95d 100644 --- a/test/ingest/evaluation-discovery/discover-prereview-evaluations.test.ts +++ b/test/ingest/evaluation-discovery/discover-prereview-evaluations.test.ts @@ -1,5 +1,6 @@ import * as TE from 'fp-ts/TaskEither'; import { pipe } from 'fp-ts/function'; +import { arbitraryIngestDays } from './ingest-days.helper'; import { discoverPrereviewEvaluations } from '../../../src/ingest/evaluation-discovery/discover-prereview-evaluations'; import { DiscoveredPublishedEvaluations } from '../../../src/ingest/types/discovered-published-evaluations'; import { constructPublishedEvaluation } from '../../../src/ingest/types/published-evaluation'; @@ -10,7 +11,7 @@ import { arbitraryArticleId } from '../../types/article-id.helper'; const runDiscovery = (stubbedResponse: unknown) => pipe( ({ fetchData: () => TE.right({ data: stubbedResponse } as unknown as D) }), - discoverPrereviewEvaluations(), + discoverPrereviewEvaluations()(arbitraryIngestDays()), ); describe('discover-prereview-evaluations', () => { diff --git a/test/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.test.ts b/test/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.test.ts index d2b4cb5652..27cf6bd6df 100644 --- a/test/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.test.ts +++ b/test/ingest/evaluation-discovery/discover-rapid-reviews-evaluations.test.ts @@ -2,6 +2,7 @@ import * as E from 'fp-ts/Either'; import * as T from 'fp-ts/Task'; import * as TE from 'fp-ts/TaskEither'; import { pipe } from 'fp-ts/function'; +import { arbitraryIngestDays } from './ingest-days.helper'; import { discoverRapidReviewsEvaluations } from '../../../src/ingest/evaluation-discovery/discover-rapid-reviews-evaluations'; import { DiscoveredPublishedEvaluations } from '../../../src/ingest/types/discovered-published-evaluations'; import { constructPublishedEvaluation } from '../../../src/ingest/types/published-evaluation'; @@ -14,7 +15,7 @@ const ingest = (crossrefResponseItems: ReadonlyArray) => pipe( fetchData: () => TE.right({ message: { items: crossrefResponseItems } } as unknown as D), fetchGoogleSheet: shouldNotBeCalled, }, - discoverRapidReviewsEvaluations(), + discoverRapidReviewsEvaluations()(arbitraryIngestDays()), ); describe('discover-rapid-reviews-evaluations', () => { diff --git a/test/ingest/evaluation-discovery/ingest-days.helper.ts b/test/ingest/evaluation-discovery/ingest-days.helper.ts new file mode 100644 index 0000000000..2a85d28a12 --- /dev/null +++ b/test/ingest/evaluation-discovery/ingest-days.helper.ts @@ -0,0 +1 @@ +export const arbitraryIngestDays = (): number => 5;