From 060d6adbe4dc2ac1758f56d419aae81f29a51e6d Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Wed, 1 Apr 2020 10:50:05 -0400 Subject: [PATCH 01/12] WIP: initial set up for clinicaltrials.gov translator --- clinicaltrials.gov.js | 139 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 clinicaltrials.gov.js diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js new file mode 100644 index 0000000000..8b3902b06c --- /dev/null +++ b/clinicaltrials.gov.js @@ -0,0 +1,139 @@ +{ + "translatorID": "874d70a0-6b95-4391-a681-c56dabaa1411", + "label": "Clinical Trials", + "creator": "Ryan Velazquez", + "target": "^https://(www\\.)?clinicaltrials\\.gov/", + "minVersion": "3.0", + "maxVersion": "", + "priority": 100, + "inRepository": true, + "translatorType": 4, // TODO + "browserSupport": "gcsibv", // TODO + "lastUpdated": "2020-04-01 11:00:00" +} + +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2020 Ryan Velazquez + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +function detectWeb(doc, url) { + // TODO: confirm that there all searches will have ct2/results in the url + if (url.includes('https://clinicaltrials.gov/ct2/results')){ + return "multiple" + } else { + return "journalArticle" + } +} + +// TODO: implement the search functionality +function getSearchResults(doc, checkOnly) { + return false +} + +function doWeb(doc, url) { + if (detectWeb(doc, url) == "multiple") { + return null // TODO: implement the search functionality, just returning null for now + Zotero.selectItems(getSearchResults(doc, false), function (items) { + if (items) ZU.processDocuments(Object.keys(items), scrape); + }); + } + else { + scrape(doc, url); + } +} + +function isJsonAPIRequest(url) { + if (url.includes("https://clinicaltrials.gov/api/query") && url.includes("fmt=JSON")) { + return true + } else { + return false + } +} + +function isXmlAPIRequest(url) { + if (url.includes("https://clinicaltrials.gov/api/query") && url.includes("fmt=XML")) { + return true + } else { + return false + } +} + +function getClinicalTrialID(url) { + // TODO: make sure this handles all the potential URLs + if (isXmlAPIRequest(url)){ + return url.split("expr=")[1].split("&")[0] + } else { + return url.split('/show/')[1] + } +} + +function dateTimeToDateString(dateTime) { + return dateTime.split(" ")[0].split(":").join("-") +} + +function scrape(doc, url) { + let jsonRequestURL + if (!isJsonAPIRequest(url)){ + const clinicalTrialID = getClinicalTrialID(url) + jsonRequestURL = "https://clinicaltrials.gov/api/query/full_studies?expr=" + clinicalTrialID + "&fmt=JSON" + } else { + jsonRequestURL = url + } + + https.get(jsonRequestURL, (resp) => { // TODO: replace the `https.get` with `ZU.doGet` and modify accordingly + let data = '' + resp.on('data', (chunk) => { data += chunk; }); + resp.on('end', () => { + var item = {} /// TODO: replace `var item = {}` with `var item = new Zotero.Item(type)` + data = JSON.parse(data) + item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs) + item.title = data.FullStudiesResponse.FullStudies[0].Study.ProtocolSection.IdentificationModule.OfficialTitle + item.shortTitle = data.FullStudiesResponse.FullStudies[0].Study.ProtocolSection.IdentificationModule.BriefTitle + // TODO: parse all the data from the json into the `item` + console.log('item: ', item); + }) + }) +} + + +/** BEGIN TEST CASES **/ +var testCases = [ // TODO: set up test cases + +] +/** END TEST CASES **/ + + +// TEMPORARY: Test Function for local testing with node while developing +// comment out the json metadata at the top and then run `node clinicaltrials.gov.js` + +const https = require('https') + +function testURL(url, testName){ + console.log('\n ---- testing ', testName) + doWebResult = doWeb(null, url) +} + +console.log('running temporary tests') +testURL("https://clinicaltrials.gov/ct2/show/NCT04292899", "clinical trials main web page") +testURL("https://clinicaltrials.gov/api/query/full_studies?expr=NCT04292899&fmt=JSON", "clinical trials json api request") +testURL("https://clinicaltrials.gov/api/query/full_studies?expr=NCT04292899&fmt=XML", "clinical trials xml api request") +testURL("https://clinicaltrials.gov/ct2/results?recrs=ab&cond=COVID-19&term=&cntry=&state=&city=&dist=", "clinical trials search result") From 00ea317e2bc23dffac4145e32181ac213e9c3157 Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Wed, 1 Apr 2020 13:20:32 -0400 Subject: [PATCH 02/12] use report item type; add test case; parse the item data; try using Zotero modules --- clinicaltrials.gov.js | 90 +++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 38 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 8b3902b06c..9c9ac4127c 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -36,11 +36,10 @@ */ function detectWeb(doc, url) { - // TODO: confirm that there all searches will have ct2/results in the url if (url.includes('https://clinicaltrials.gov/ct2/results')){ return "multiple" } else { - return "journalArticle" + return "report" } } @@ -79,10 +78,10 @@ function isXmlAPIRequest(url) { function getClinicalTrialID(url) { // TODO: make sure this handles all the potential URLs - if (isXmlAPIRequest(url)){ + if (isXmlAPIRequest(url) || isJsonAPIRequest(url)){ return url.split("expr=")[1].split("&")[0] } else { - return url.split('/show/')[1] + return url.split('/show/')[1].split("?")[0] } } @@ -91,49 +90,64 @@ function dateTimeToDateString(dateTime) { } function scrape(doc, url) { + const clinicalTrialID = getClinicalTrialID(url) let jsonRequestURL if (!isJsonAPIRequest(url)){ - const clinicalTrialID = getClinicalTrialID(url) - jsonRequestURL = "https://clinicaltrials.gov/api/query/full_studies?expr=" + clinicalTrialID + "&fmt=JSON" + jsonRequestURL = `https://clinicaltrials.gov/api/query/full_studies?expr=${clinicalTrialID}&fmt=JSON` } else { jsonRequestURL = url } - https.get(jsonRequestURL, (resp) => { // TODO: replace the `https.get` with `ZU.doGet` and modify accordingly - let data = '' - resp.on('data', (chunk) => { data += chunk; }); - resp.on('end', () => { - var item = {} /// TODO: replace `var item = {}` with `var item = new Zotero.Item(type)` - data = JSON.parse(data) - item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs) - item.title = data.FullStudiesResponse.FullStudies[0].Study.ProtocolSection.IdentificationModule.OfficialTitle - item.shortTitle = data.FullStudiesResponse.FullStudies[0].Study.ProtocolSection.IdentificationModule.BriefTitle - // TODO: parse all the data from the json into the `item` - console.log('item: ', item); - }) + ZU.doGet(jsonRequestURL, function(resp) { + data = JSON.parse(resp) + var item = new Zotero.Item(type) + const study = data.FullStudiesResponse.FullStudies[0].Study + item.itemType = "report" + item.title = study.ProtocolSection.IdentificationModule.OfficialTitle + + // Get the creator info + const responsibleParty = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty + if (typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string") { + let authorName = responsibleParty.ResponsiblePartyInvestigatorFullName + item.creator = { + author: authorName + } + } else if (study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty.ResponsiblePartyType == "Sponsor") { + let sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName + item.creator = { + author: sponsor + } + } + + item.date = study.ProtocolSection.StatusModule.LastUpdateSubmitDate // TODO: is this the date that we want? Would "StudyFirstSubmitDate" be better? + item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs) + item.libraryCatalog = "clinicaltrials.gov" + item.shortTitle = study.ProtocolSection.IdentificationModule.BriefTitle + item.url = "https://clinicaltrials.gov/ct2/show/" + clinicalTrialID + item.complete() }) } /** BEGIN TEST CASES **/ -var testCases = [ // TODO: set up test cases - +var testCases = [ // TODO: set up more test cases + { + "type": "web", + "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", + "items": [ + { + "itemType": "report", + "title": "A Phase 3 Randomized Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe COVID-19", + "creator": { + "author": "Gilead Sciences" + }, + "date": "February 28, 2020", + "accessDate": "2020-04-01", + "libraryCatalog": "clinicaltrials.gov", + "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", + "url": "https://clinicaltrials.gov/ct2/show/NCT04292899" + } + ] + } ] -/** END TEST CASES **/ - - -// TEMPORARY: Test Function for local testing with node while developing -// comment out the json metadata at the top and then run `node clinicaltrials.gov.js` - -const https = require('https') - -function testURL(url, testName){ - console.log('\n ---- testing ', testName) - doWebResult = doWeb(null, url) -} - -console.log('running temporary tests') -testURL("https://clinicaltrials.gov/ct2/show/NCT04292899", "clinical trials main web page") -testURL("https://clinicaltrials.gov/api/query/full_studies?expr=NCT04292899&fmt=JSON", "clinical trials json api request") -testURL("https://clinicaltrials.gov/api/query/full_studies?expr=NCT04292899&fmt=XML", "clinical trials xml api request") -testURL("https://clinicaltrials.gov/ct2/results?recrs=ab&cond=COVID-19&term=&cntry=&state=&city=&dist=", "clinical trials search result") +/** END TEST CASES **/ \ No newline at end of file From 30d16765f2a67419b79a89a257899e95e133b2b9 Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Wed, 1 Apr 2020 13:50:38 -0400 Subject: [PATCH 03/12] fix lint errors and warnings and remove search code for now --- clinicaltrials.gov.js | 118 ++++++++++++++++++++---------------------- 1 file changed, 56 insertions(+), 62 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 9c9ac4127c..13b6b13963 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -1,15 +1,15 @@ { "translatorID": "874d70a0-6b95-4391-a681-c56dabaa1411", - "label": "Clinical Trials", + "label": "clinicaltrials.gov", "creator": "Ryan Velazquez", "target": "^https://(www\\.)?clinicaltrials\\.gov/", "minVersion": "3.0", "maxVersion": "", "priority": 100, "inRepository": true, - "translatorType": 4, // TODO - "browserSupport": "gcsibv", // TODO - "lastUpdated": "2020-04-01 11:00:00" + "translatorType": 4, + "browserSupport": "gcsibv", + "lastUpdated": "2020-04-01 14:00:00" } /* @@ -36,101 +36,95 @@ */ function detectWeb(doc, url) { - if (url.includes('https://clinicaltrials.gov/ct2/results')){ - return "multiple" - } else { - return "report" + if (url.includes('https://clinicaltrials.gov/ct2/results')) { + throw new Error('clinicaltrials.gov search pages not supported by Zotero, only individual trials'); + } + else { + return "report"; } -} - -// TODO: implement the search functionality -function getSearchResults(doc, checkOnly) { - return false } function doWeb(doc, url) { - if (detectWeb(doc, url) == "multiple") { - return null // TODO: implement the search functionality, just returning null for now - Zotero.selectItems(getSearchResults(doc, false), function (items) { - if (items) ZU.processDocuments(Object.keys(items), scrape); - }); - } - else { - scrape(doc, url); - } + scrape(doc, url); } function isJsonAPIRequest(url) { if (url.includes("https://clinicaltrials.gov/api/query") && url.includes("fmt=JSON")) { - return true - } else { - return false + return true; + } + else { + return false; } } function isXmlAPIRequest(url) { if (url.includes("https://clinicaltrials.gov/api/query") && url.includes("fmt=XML")) { - return true - } else { - return false + return true; + } + else { + return false; } } function getClinicalTrialID(url) { // TODO: make sure this handles all the potential URLs - if (isXmlAPIRequest(url) || isJsonAPIRequest(url)){ - return url.split("expr=")[1].split("&")[0] - } else { - return url.split('/show/')[1].split("?")[0] + if (isXmlAPIRequest(url) || isJsonAPIRequest(url)) { + return url.split("expr=")[1].split("&")[0]; + } + else { + return url.split('/show/')[1].split("?")[0]; } } function dateTimeToDateString(dateTime) { - return dateTime.split(" ")[0].split(":").join("-") + return dateTime.split(" ")[0].split(":").join("-"); } function scrape(doc, url) { - const clinicalTrialID = getClinicalTrialID(url) - let jsonRequestURL - if (!isJsonAPIRequest(url)){ - jsonRequestURL = `https://clinicaltrials.gov/api/query/full_studies?expr=${clinicalTrialID}&fmt=JSON` - } else { - jsonRequestURL = url + const clinicalTrialID = getClinicalTrialID(url); + let jsonRequestURL; + if (!isJsonAPIRequest(url)) { + jsonRequestURL = `https://clinicaltrials.gov/api/query/full_studies?expr=${clinicalTrialID}&fmt=JSON`; + } + else { + jsonRequestURL = url; } - ZU.doGet(jsonRequestURL, function(resp) { - data = JSON.parse(resp) - var item = new Zotero.Item(type) - const study = data.FullStudiesResponse.FullStudies[0].Study - item.itemType = "report" - item.title = study.ProtocolSection.IdentificationModule.OfficialTitle + ZU.doGet(jsonRequestURL, function (resp) { + const data = JSON.parse(resp); + var item = new Zotero.Item("report"); + const study = data.FullStudiesResponse.FullStudies[0].Study; + item.itemType = "report"; + item.title = study.ProtocolSection.IdentificationModule.OfficialTitle; // Get the creator info - const responsibleParty = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty + const responsibleParty = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty; if (typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string") { - let authorName = responsibleParty.ResponsiblePartyInvestigatorFullName + let authorName = responsibleParty.ResponsiblePartyInvestigatorFullName; item.creator = { author: authorName - } - } else if (study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty.ResponsiblePartyType == "Sponsor") { - let sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName + }; + } + else if (study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty.ResponsiblePartyType == "Sponsor") { + let sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName; item.creator = { author: sponsor - } + }; } - item.date = study.ProtocolSection.StatusModule.LastUpdateSubmitDate // TODO: is this the date that we want? Would "StudyFirstSubmitDate" be better? - item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs) - item.libraryCatalog = "clinicaltrials.gov" - item.shortTitle = study.ProtocolSection.IdentificationModule.BriefTitle - item.url = "https://clinicaltrials.gov/ct2/show/" + clinicalTrialID - item.complete() - }) + item.date = study.ProtocolSection.StatusModule.LastUpdateSubmitDate; // TODO: is this the date that we want? Would "StudyFirstSubmitDate" be better? + item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs); + item.libraryCatalog = "clinicaltrials.gov"; + item.shortTitle = study.ProtocolSection.IdentificationModule.BriefTitle; + item.url = "https://clinicaltrials.gov/ct2/show/" + clinicalTrialID; + item.complete(); + }); } /** BEGIN TEST CASES **/ -var testCases = [ // TODO: set up more test cases +// TODO: set up more test cases +var testCases = [ { "type": "web", "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", @@ -139,8 +133,8 @@ var testCases = [ // TODO: set up more test cases "itemType": "report", "title": "A Phase 3 Randomized Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe COVID-19", "creator": { - "author": "Gilead Sciences" - }, + "author": "Gilead Sciences" + }, "date": "February 28, 2020", "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", @@ -150,4 +144,4 @@ var testCases = [ // TODO: set up more test cases ] } ] -/** END TEST CASES **/ \ No newline at end of file +/** END TEST CASES **/ From ff3a1a3e11a2f0db56ad684ac3043b841e16b01d Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Wed, 1 Apr 2020 20:40:01 -0400 Subject: [PATCH 04/12] address comments; additional test cases --- clinicaltrials.gov.js | 162 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 140 insertions(+), 22 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 13b6b13963..da65656d06 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -36,8 +36,8 @@ */ function detectWeb(doc, url) { - if (url.includes('https://clinicaltrials.gov/ct2/results')) { - throw new Error('clinicaltrials.gov search pages not supported by Zotero, only individual trials'); + if (url.includes("https://clinicaltrials.gov/ct2/results")) { + throw new Error("clinicaltrials.gov search pages not supported by Zotero, only individual trials"); } else { return "report"; @@ -67,12 +67,11 @@ function isXmlAPIRequest(url) { } function getClinicalTrialID(url) { - // TODO: make sure this handles all the potential URLs if (isXmlAPIRequest(url) || isJsonAPIRequest(url)) { return url.split("expr=")[1].split("&")[0]; } else { - return url.split('/show/')[1].split("?")[0]; + return url.split("/show/")[1].split("?")[0]; } } @@ -80,6 +79,13 @@ function dateTimeToDateString(dateTime) { return dateTime.split(" ")[0].split(":").join("-"); } +function nameToFirstAndLast(rawName) { + const name = rawName.split(",")[0]; + const firstName = name.split(" ")[0]; + const lastName = name.split(",")[0].split(" ")[name.split(" ").length - 1]; + return [firstName, lastName]; +} + function scrape(doc, url) { const clinicalTrialID = getClinicalTrialID(url); let jsonRequestURL; @@ -97,33 +103,74 @@ function scrape(doc, url) { item.itemType = "report"; item.title = study.ProtocolSection.IdentificationModule.OfficialTitle; - // Get the creator info - const responsibleParty = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty; - if (typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string") { - let authorName = responsibleParty.ResponsiblePartyInvestigatorFullName; - item.creator = { - author: authorName - }; + // Start get the creator info + let creators = []; + let responsiblePartyInvestigator; + let sponsor; + let collaborators = []; + if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("ResponsibleParty")) { + const responsibleParty = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty; + if (typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string") { + responsiblePartyInvestigator = responsibleParty.ResponsiblePartyInvestigatorFullName; + const splitResponsiblePartyInvestigator = nameToFirstAndLast(responsiblePartyInvestigator); + creators.push({ + firstName: splitResponsiblePartyInvestigator[0], + lastName: splitResponsiblePartyInvestigator[1], + creatorType: "author" + }); + } } - else if (study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty.ResponsiblePartyType == "Sponsor") { - let sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName; - item.creator = { - author: sponsor - }; + + if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("LeadSponsor")) { + sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName; + creators.push({ + firstName: sponsor, + creatorType: "author" + }); } + + if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("CollaboratorList")) { + const collaboratorList = study.ProtocolSection.SponsorCollaboratorsModule.CollaboratorList.Collaborator; + collaboratorList.forEach((collaborator) => { + collaborators.push( + { + firstName: collaborator.CollaboratorName, + creatorType: "author" + } + ); + }); + collaborators.forEach((collaborator) => { + creators.push(collaborator); + }); + } + + item.creators = creators; + // Done get the creator info - item.date = study.ProtocolSection.StatusModule.LastUpdateSubmitDate; // TODO: is this the date that we want? Would "StudyFirstSubmitDate" be better? + item.date = study.ProtocolSection.StatusModule.LastUpdateSubmitDate; item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs); item.libraryCatalog = "clinicaltrials.gov"; item.shortTitle = study.ProtocolSection.IdentificationModule.BriefTitle; item.url = "https://clinicaltrials.gov/ct2/show/" + clinicalTrialID; + let extras = { + submittedDate: study.ProtocolSection.StatusModule.StudyFirstSubmitDate, + responsiblePartyInvestigator: responsiblePartyInvestigator, + sponsor: sponsor + }; + if (collaborators.length > 0) { + extras.collaborators = []; + collaborators.forEach((collaborator) => { + extras.collaborators.push(collaborator.firstName); + }); + } + + item.extra = extras; item.complete(); }); } /** BEGIN TEST CASES **/ -// TODO: set up more test cases var testCases = [ { "type": "web", @@ -132,16 +179,87 @@ var testCases = [ { "itemType": "report", "title": "A Phase 3 Randomized Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe COVID-19", - "creator": { - "author": "Gilead Sciences" - }, + "creators": [ + { + "firstName": "Gilead Sciences", + "creatorType": "author" + } + ], "date": "February 28, 2020", "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", - "url": "https://clinicaltrials.gov/ct2/show/NCT04292899" + "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", + "extra": { + "submittedDate": "February 6, 2020", + "responsiblePartyInvestigator": "undefined", + "sponsor": "Gilead Sciences" + } + } + ] + }, + { + "type": "web", + "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", + "items": [ + { + "itemType": "report", + "title": "The Impact of Gastroesophageal Reflux Disease in Sleep Disorders: A Pilot Investigation of Rabeprazole, 20 mg Twice Daily for the Relief of GERD-Related Insomnia.", + "creators": [ + { + "firstName": "University of North Carolina", + "creatorType": "author" + }, + { + "firstName": "Janssen Pharmaceutica N.V., Belgium", + "creatorType": "author" + } + ], + "date": "April 25, 2007", + "accessDate": "2020-04-01", + "libraryCatalog": "clinicaltrials.gov", + "shortTitle": "Sleep Disorders and Gastroesophageal Reflux Disease (GERD)", + "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", + "extra": { + "submittedDate": "February 3, 2006", + "responsiblePartyInvestigator": "undefined", + "sponsor": "University of North Carolina", + "collaborators": ["Janssen Pharmaceutica N.V., Belgium"] + } + } + ] + }, + { + "type": "web", + "url": "https://clinicaltrials.gov/ct2/show/NCT04261517?recrs=e&cond=COVID&draw=2&rank=8", + "items": [ + { + "itemType": "report", + "title": "Efficacy and Safety of Hydroxychloroquine for Treatment of Pneumonia Caused by 2019-nCoV ( HC-nCoV )", + "creators": [ + { + "firstName": "Hongzhou", + "lastName": "Lu", + "creatorType": "author" + }, + { + "firstName": "Shanghai Public Health Clinical Center", + "creatorType": "author" + } + ], + "date": "March 22, 2020", + "accessDate": "2020-04-01", + "libraryCatalog": "clinicaltrials.gov", + "shortTitle": "Efficacy and Safety of Hydroxychloroquine for Treatment of Pneumonia Caused by 2019-nCoV ( HC-nCoV )", + "url": "https://clinicaltrials.gov/ct2/show/NCT04261517", + "extra": { + "submittedDate": "February 6, 2020", + "responsiblePartyInvestigator": "Hongzhou Lu", + "sponsor": "Shanghai Public Health Clinical Center" + } } ] } + ] /** END TEST CASES **/ From 5f32f2b37272281ea66316918482682874484155 Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Wed, 1 Apr 2020 22:18:08 -0400 Subject: [PATCH 05/12] extra as new line delimited string --- clinicaltrials.gov.js | 52 ++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index da65656d06..64f3f29a06 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -86,6 +86,20 @@ function nameToFirstAndLast(rawName) { return [firstName, lastName]; } +function extrasObjToExtrasString(extrasObj) { + let extrasString = ""; + for (let key in extrasObj) { + if (key == "collaborators") { + const stringifiedArray = "'" + extrasObj[key].join("','") + "'"; + extrasString = extrasString + "\n" + key + ": " + stringifiedArray; + } + else { + extrasString = extrasString + "\n" + key + ": " + extrasObj[key]; + } + } + return extrasString; +} + function scrape(doc, url) { const clinicalTrialID = getClinicalTrialID(url); let jsonRequestURL; @@ -163,8 +177,8 @@ function scrape(doc, url) { extras.collaborators.push(collaborator.firstName); }); } - - item.extra = extras; + const extrasString = extrasObjToExtrasString(extras); + item.extra = extrasString; item.complete(); }); } @@ -185,16 +199,16 @@ var testCases = [ "creatorType": "author" } ], - "date": "February 28, 2020", + "date": "March 27, 2020", "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", - "extra": { - "submittedDate": "February 6, 2020", - "responsiblePartyInvestigator": "undefined", - "sponsor": "Gilead Sciences" - } + "extra": "submittedDate: February 28, 2020\nresponsiblePartyInvestigator: undefined\nsponsor: Gilead Sciences", + "notes": [], + "tags": [], + "seeAlso": [], + "attachments": [] } ] }, @@ -220,12 +234,11 @@ var testCases = [ "libraryCatalog": "clinicaltrials.gov", "shortTitle": "Sleep Disorders and Gastroesophageal Reflux Disease (GERD)", "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", - "extra": { - "submittedDate": "February 3, 2006", - "responsiblePartyInvestigator": "undefined", - "sponsor": "University of North Carolina", - "collaborators": ["Janssen Pharmaceutica N.V., Belgium"] - } + "extra": "submittedDate: February 3, 2006\nresponsiblePartyInvestigator: undefined\nsponsor: University of North Carolina\ncollaborators: 'Janssen Pharmaceutica N.V., Belgium'", + "notes": [], + "tags": [], + "seeAlso": [], + "attachments": [] } ] }, @@ -250,13 +263,12 @@ var testCases = [ "date": "March 22, 2020", "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", - "shortTitle": "Efficacy and Safety of Hydroxychloroquine for Treatment of Pneumonia Caused by 2019-nCoV ( HC-nCoV )", "url": "https://clinicaltrials.gov/ct2/show/NCT04261517", - "extra": { - "submittedDate": "February 6, 2020", - "responsiblePartyInvestigator": "Hongzhou Lu", - "sponsor": "Shanghai Public Health Clinical Center" - } + "extra": "submittedDate: February 6, 2020\nresponsiblePartyInvestigator: Hongzhou Lu\nsponsor: Shanghai Public Health Clinical Center", + "notes": [], + "tags": [], + "seeAlso": [], + "attachments": [] } ] } From 0d1e83726f5b751817136095801ef2fec95609a1 Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Thu, 2 Apr 2020 14:56:01 -0400 Subject: [PATCH 06/12] address all @bwiernik comments excpet for creators --- clinicaltrials.gov.js | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 64f3f29a06..40d1bec098 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -163,14 +163,20 @@ function scrape(doc, url) { item.date = study.ProtocolSection.StatusModule.LastUpdateSubmitDate; item.accessDate = dateTimeToDateString(data.FullStudiesResponse.DataVrs); - item.libraryCatalog = "clinicaltrials.gov"; + item.institution = "clinicaltrials.gov"; + item.reportNumber = clinicalTrialID; item.shortTitle = study.ProtocolSection.IdentificationModule.BriefTitle; + item.abstractNote = study.ProtocolSection.DescriptionModule.BriefSummary; item.url = "https://clinicaltrials.gov/ct2/show/" + clinicalTrialID; + item.reportType = "Clinical trial registration"; + let extras = { - submittedDate: study.ProtocolSection.StatusModule.StudyFirstSubmitDate, - responsiblePartyInvestigator: responsiblePartyInvestigator, + submitted: study.ProtocolSection.StatusModule.StudyFirstSubmitDate, sponsor: sponsor }; + if (responsiblePartyInvestigator) { + extras["Principle investigator"] = responsiblePartyInvestigator; + } if (collaborators.length > 0) { extras.collaborators = []; collaborators.forEach((collaborator) => { @@ -193,18 +199,22 @@ var testCases = [ { "itemType": "report", "title": "A Phase 3 Randomized Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe COVID-19", + "abstractNote": "The primary objective of this study is to evaluate the efficacy of 2 remdesivir (RDV) regimens with respect to the normalization of temperature and oxygen saturation through Day 14 in participants with severe coronavirus disease (COVID-19).", "creators": [ { "firstName": "Gilead Sciences", "creatorType": "author" } ], - "date": "March 27, 2020", + "date": "April 1, 2020", "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", - "extra": "submittedDate: February 28, 2020\nresponsiblePartyInvestigator: undefined\nsponsor: Gilead Sciences", + "institution": "clinicaltrials.gov", + "reportNumber": "NCT04292899", + "reportType": "Clinical trial registration", + "extra": "submitted: February 28, 2020\nsponsor: Gilead Sciences", "notes": [], "tags": [], "seeAlso": [], @@ -219,6 +229,7 @@ var testCases = [ { "itemType": "report", "title": "The Impact of Gastroesophageal Reflux Disease in Sleep Disorders: A Pilot Investigation of Rabeprazole, 20 mg Twice Daily for the Relief of GERD-Related Insomnia.", + "abstractNote": "This study will investigate Gastroesophageal Reflux Disease (GERD)as a cause of sleep disturbance. Patients with GERD may experience all or some of the following symptoms: stomach acid or partially digested food re-entering the esophagus (which is sometimes referred to as heartburn or regurgitation) and belching. Even very small, unnoticeable amounts of rising stomach acid may cause patients to wake up during the night.\n\nThis study will also investigate the effect of Rabeprazole, (brand name Aciphex) on patients with known insomnia. Rabeprazole is an FDA approved medication already marketed for the treatment of GERD.", "creators": [ { "firstName": "University of North Carolina", @@ -234,7 +245,10 @@ var testCases = [ "libraryCatalog": "clinicaltrials.gov", "shortTitle": "Sleep Disorders and Gastroesophageal Reflux Disease (GERD)", "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", - "extra": "submittedDate: February 3, 2006\nresponsiblePartyInvestigator: undefined\nsponsor: University of North Carolina\ncollaborators: 'Janssen Pharmaceutica N.V., Belgium'", + "institution": "clinicaltrials.gov", + "reportNumber": "NCT00287391", + "reportType": "Clinical trial registration", + "extra": "submitted: February 3, 2006\nsponsor: University of North Carolina\ncollaborators: 'Janssen Pharmaceutica N.V., Belgium'", "notes": [], "tags": [], "seeAlso": [], @@ -249,6 +263,7 @@ var testCases = [ { "itemType": "report", "title": "Efficacy and Safety of Hydroxychloroquine for Treatment of Pneumonia Caused by 2019-nCoV ( HC-nCoV )", + "abstractNote": "The study aims to evaluate the efficacy and safety of hydroxychloroquine in the treatment of pneumonia caused by the 2019 novel coronavirus", "creators": [ { "firstName": "Hongzhou", @@ -264,7 +279,10 @@ var testCases = [ "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", "url": "https://clinicaltrials.gov/ct2/show/NCT04261517", - "extra": "submittedDate: February 6, 2020\nresponsiblePartyInvestigator: Hongzhou Lu\nsponsor: Shanghai Public Health Clinical Center", + "institution": "clinicaltrials.gov", + "reportNumber": "NCT04261517", + "reportType": "Clinical trial registration", + "extra": "submitted: February 6, 2020\nsponsor: Shanghai Public Health Clinical Center\nPrinciple investigator: Hongzhou Lu", "notes": [], "tags": [], "seeAlso": [], From 1d5177eb47ef937d465b6d51b36b52b30d2002fb Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Thu, 2 Apr 2020 19:26:28 -0400 Subject: [PATCH 07/12] creator edits; remove creators from extra --- clinicaltrials.gov.js | 52 +++++++++++++------------------------------ 1 file changed, 15 insertions(+), 37 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 40d1bec098..3a082818ce 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -86,20 +86,6 @@ function nameToFirstAndLast(rawName) { return [firstName, lastName]; } -function extrasObjToExtrasString(extrasObj) { - let extrasString = ""; - for (let key in extrasObj) { - if (key == "collaborators") { - const stringifiedArray = "'" + extrasObj[key].join("','") + "'"; - extrasString = extrasString + "\n" + key + ": " + stringifiedArray; - } - else { - extrasString = extrasString + "\n" + key + ": " + extrasObj[key]; - } - } - return extrasString; -} - function scrape(doc, url) { const clinicalTrialID = getClinicalTrialID(url); let jsonRequestURL; @@ -137,9 +123,16 @@ function scrape(doc, url) { if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("LeadSponsor")) { sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName; + let sponsorCreatorType; + if (creators.length == 0){ + sponsorCreatorType = "author"; + } + else { + sponsorCreatorType = "contributor" + } creators.push({ firstName: sponsor, - creatorType: "author" + creatorType: sponsorCreatorType }); } @@ -149,7 +142,7 @@ function scrape(doc, url) { collaborators.push( { firstName: collaborator.CollaboratorName, - creatorType: "author" + creatorType: "contributor" } ); }); @@ -169,22 +162,7 @@ function scrape(doc, url) { item.abstractNote = study.ProtocolSection.DescriptionModule.BriefSummary; item.url = "https://clinicaltrials.gov/ct2/show/" + clinicalTrialID; item.reportType = "Clinical trial registration"; - - let extras = { - submitted: study.ProtocolSection.StatusModule.StudyFirstSubmitDate, - sponsor: sponsor - }; - if (responsiblePartyInvestigator) { - extras["Principle investigator"] = responsiblePartyInvestigator; - } - if (collaborators.length > 0) { - extras.collaborators = []; - collaborators.forEach((collaborator) => { - extras.collaborators.push(collaborator.firstName); - }); - } - const extrasString = extrasObjToExtrasString(extras); - item.extra = extrasString; + item.extra = `submitted: ${study.ProtocolSection.StatusModule.StudyFirstSubmitDate}`; item.complete(); }); } @@ -214,7 +192,7 @@ var testCases = [ "institution": "clinicaltrials.gov", "reportNumber": "NCT04292899", "reportType": "Clinical trial registration", - "extra": "submitted: February 28, 2020\nsponsor: Gilead Sciences", + "extra": "submitted: February 28, 2020", "notes": [], "tags": [], "seeAlso": [], @@ -237,7 +215,7 @@ var testCases = [ }, { "firstName": "Janssen Pharmaceutica N.V., Belgium", - "creatorType": "author" + "creatorType": "contributor" } ], "date": "April 25, 2007", @@ -248,7 +226,7 @@ var testCases = [ "institution": "clinicaltrials.gov", "reportNumber": "NCT00287391", "reportType": "Clinical trial registration", - "extra": "submitted: February 3, 2006\nsponsor: University of North Carolina\ncollaborators: 'Janssen Pharmaceutica N.V., Belgium'", + "extra": "submitted: February 3, 2006", "notes": [], "tags": [], "seeAlso": [], @@ -272,7 +250,7 @@ var testCases = [ }, { "firstName": "Shanghai Public Health Clinical Center", - "creatorType": "author" + "creatorType": "contributor" } ], "date": "March 22, 2020", @@ -282,7 +260,7 @@ var testCases = [ "institution": "clinicaltrials.gov", "reportNumber": "NCT04261517", "reportType": "Clinical trial registration", - "extra": "submitted: February 6, 2020\nsponsor: Shanghai Public Health Clinical Center\nPrinciple investigator: Hongzhou Lu", + "extra": "submitted: February 6, 2020", "notes": [], "tags": [], "seeAlso": [], From 06fe3fc01c2db8ba924806a24ff9f860a6f500be Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Thu, 2 Apr 2020 19:28:27 -0400 Subject: [PATCH 08/12] linter fixes --- clinicaltrials.gov.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 3a082818ce..14cc6dcb92 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -124,11 +124,11 @@ function scrape(doc, url) { if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("LeadSponsor")) { sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName; let sponsorCreatorType; - if (creators.length == 0){ + if (creators.length == 0) { sponsorCreatorType = "author"; } else { - sponsorCreatorType = "contributor" + sponsorCreatorType = "contributor"; } creators.push({ firstName: sponsor, From df916258d6d58db18c084a2c48b15bbc77ea4d6e Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Mon, 4 May 2020 09:16:33 -0400 Subject: [PATCH 09/12] more restrictive target; use ZU.cleanAuthor; fix test indenting --- clinicaltrials.gov.js | 66 ++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 42 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 14cc6dcb92..c3bba61f11 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -2,7 +2,7 @@ "translatorID": "874d70a0-6b95-4391-a681-c56dabaa1411", "label": "clinicaltrials.gov", "creator": "Ryan Velazquez", - "target": "^https://(www\\.)?clinicaltrials\\.gov/", + "target": "^https://(www\\.)?clinicaltrials\\.gov/ct2/show", "minVersion": "3.0", "maxVersion": "", "priority": 100, @@ -35,13 +35,8 @@ ***** END LICENSE BLOCK ***** */ -function detectWeb(doc, url) { - if (url.includes("https://clinicaltrials.gov/ct2/results")) { - throw new Error("clinicaltrials.gov search pages not supported by Zotero, only individual trials"); - } - else { - return "report"; - } +function detectWeb() { + return "report"; } function doWeb(doc, url) { @@ -79,13 +74,6 @@ function dateTimeToDateString(dateTime) { return dateTime.split(" ")[0].split(":").join("-"); } -function nameToFirstAndLast(rawName) { - const name = rawName.split(",")[0]; - const firstName = name.split(" ")[0]; - const lastName = name.split(",")[0].split(" ")[name.split(" ").length - 1]; - return [firstName, lastName]; -} - function scrape(doc, url) { const clinicalTrialID = getClinicalTrialID(url); let jsonRequestURL; @@ -112,12 +100,7 @@ function scrape(doc, url) { const responsibleParty = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty; if (typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string") { responsiblePartyInvestigator = responsibleParty.ResponsiblePartyInvestigatorFullName; - const splitResponsiblePartyInvestigator = nameToFirstAndLast(responsiblePartyInvestigator); - creators.push({ - firstName: splitResponsiblePartyInvestigator[0], - lastName: splitResponsiblePartyInvestigator[1], - creatorType: "author" - }); + creators.push(ZU.cleanAuthor(responsiblePartyInvestigator, "author", false)); } } @@ -177,26 +160,26 @@ var testCases = [ { "itemType": "report", "title": "A Phase 3 Randomized Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe COVID-19", - "abstractNote": "The primary objective of this study is to evaluate the efficacy of 2 remdesivir (RDV) regimens with respect to the normalization of temperature and oxygen saturation through Day 14 in participants with severe coronavirus disease (COVID-19).", + "abstractNote": "The primary objective of this study is to evaluate the efficacy of 2 remdesivir (RDV) regimens with respect to clinical status assessed by a 7-point ordinal scale on Day 14.", "creators": [ { "firstName": "Gilead Sciences", "creatorType": "author" } ], - "date": "April 1, 2020", + "date": "April 28, 2020", "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", "institution": "clinicaltrials.gov", - "reportNumber": "NCT04292899", - "reportType": "Clinical trial registration", + "reportNumber": "NCT04292899", + "reportType": "Clinical trial registration", "extra": "submitted: February 28, 2020", "notes": [], - "tags": [], - "seeAlso": [], - "attachments": [] + "tags": [], + "seeAlso": [], + "attachments": [] } ] }, @@ -224,13 +207,13 @@ var testCases = [ "shortTitle": "Sleep Disorders and Gastroesophageal Reflux Disease (GERD)", "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", "institution": "clinicaltrials.gov", - "reportNumber": "NCT00287391", - "reportType": "Clinical trial registration", + "reportNumber": "NCT00287391", + "reportType": "Clinical trial registration", "extra": "submitted: February 3, 2006", "notes": [], - "tags": [], - "seeAlso": [], - "attachments": [] + "tags": [], + "seeAlso": [], + "attachments": [] } ] }, @@ -240,8 +223,8 @@ var testCases = [ "items": [ { "itemType": "report", - "title": "Efficacy and Safety of Hydroxychloroquine for Treatment of Pneumonia Caused by 2019-nCoV ( HC-nCoV )", - "abstractNote": "The study aims to evaluate the efficacy and safety of hydroxychloroquine in the treatment of pneumonia caused by the 2019 novel coronavirus", + "title": "Efficacy and Safety of Hydroxychloroquine for Treatment of COVID-19", + "abstractNote": "The study aims to evaluate the efficacy and safety of hydroxychloroquine in the treatment of COVID-19 pneumonia.", "creators": [ { "firstName": "Hongzhou", @@ -253,21 +236,20 @@ var testCases = [ "creatorType": "contributor" } ], - "date": "March 22, 2020", + "date": "April 9, 2020", "accessDate": "2020-04-01", "libraryCatalog": "clinicaltrials.gov", "url": "https://clinicaltrials.gov/ct2/show/NCT04261517", "institution": "clinicaltrials.gov", - "reportNumber": "NCT04261517", - "reportType": "Clinical trial registration", + "reportNumber": "NCT04261517", + "reportType": "Clinical trial registration", "extra": "submitted: February 6, 2020", "notes": [], - "tags": [], - "seeAlso": [], - "attachments": [] + "tags": [], + "seeAlso": [], + "attachments": [] } ] } - ] /** END TEST CASES **/ From e654dd56a936137b3e9608b124e0ec8ccdcd3513 Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Mon, 4 May 2020 23:16:56 -0400 Subject: [PATCH 10/12] clinicaltrials.gov add multiples test fails, so not adding, but tested successfully in Firefox --- clinicaltrials.gov.js | 88 ++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 27 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index c3bba61f11..42f0b2c05e 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -2,14 +2,14 @@ "translatorID": "874d70a0-6b95-4391-a681-c56dabaa1411", "label": "clinicaltrials.gov", "creator": "Ryan Velazquez", - "target": "^https://(www\\.)?clinicaltrials\\.gov/ct2/show", + "target": "^https://(www\\.)?clinicaltrials\\.gov/ct2/(show|results\\?)", "minVersion": "3.0", "maxVersion": "", "priority": 100, "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2020-04-01 14:00:00" + "lastUpdated": "2020-05-05 03:10:07" } /* @@ -35,14 +35,48 @@ ***** END LICENSE BLOCK ***** */ -function detectWeb() { + +function detectWeb(doc, url) { + Zotero.monitorDOMChanges(doc.querySelector('#theDataTable')); + if (url.includes("/ct2/results?") && getSearchResults(doc, true)) { + return "multiple"; + } + + if (url.includes('/ct2/show')) { return "report"; + } + + return false; +} + +function getSearchResults(doc, checkOnly) { + var items = {}; + var found = false; + var rows = doc.querySelectorAll('table#theDataTable a[href*="/ct2/show"]'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; } function doWeb(doc, url) { + if (detectWeb(doc, url) == "multiple") { + Zotero.selectItems(getSearchResults(doc, false), function (items) { + if (items) ZU.processDocuments(Object.keys(items), scrape); + }); + } else + { scrape(doc, url); + } } + + function isJsonAPIRequest(url) { if (url.includes("https://clinicaltrials.gov/api/query") && url.includes("fmt=JSON")) { return true; @@ -152,7 +186,7 @@ function scrape(doc, url) { /** BEGIN TEST CASES **/ -var testCases = [ +var testCases = [ { "type": "web", "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", @@ -160,7 +194,6 @@ var testCases = [ { "itemType": "report", "title": "A Phase 3 Randomized Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe COVID-19", - "abstractNote": "The primary objective of this study is to evaluate the efficacy of 2 remdesivir (RDV) regimens with respect to clinical status assessed by a 7-point ordinal scale on Day 14.", "creators": [ { "firstName": "Gilead Sciences", @@ -168,18 +201,19 @@ var testCases = [ } ], "date": "April 28, 2020", + "abstractNote": "The primary objective of this study is to evaluate the efficacy of 2 remdesivir (RDV) regimens with respect to clinical status assessed by a 7-point ordinal scale on Day 14.", "accessDate": "2020-04-01", - "libraryCatalog": "clinicaltrials.gov", - "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", - "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", + "extra": "submitted: February 28, 2020", "institution": "clinicaltrials.gov", + "libraryCatalog": "clinicaltrials.gov", "reportNumber": "NCT04292899", "reportType": "Clinical trial registration", - "extra": "submitted: February 28, 2020", - "notes": [], + "shortTitle": "Study to Evaluate the Safety and Antiviral Activity of Remdesivir (GS-5734™) in Participants With Severe Coronavirus Disease (COVID-19)", + "url": "https://clinicaltrials.gov/ct2/show/NCT04292899", + "attachments": [], "tags": [], - "seeAlso": [], - "attachments": [] + "notes": [], + "seeAlso": [] } ] }, @@ -190,7 +224,6 @@ var testCases = [ { "itemType": "report", "title": "The Impact of Gastroesophageal Reflux Disease in Sleep Disorders: A Pilot Investigation of Rabeprazole, 20 mg Twice Daily for the Relief of GERD-Related Insomnia.", - "abstractNote": "This study will investigate Gastroesophageal Reflux Disease (GERD)as a cause of sleep disturbance. Patients with GERD may experience all or some of the following symptoms: stomach acid or partially digested food re-entering the esophagus (which is sometimes referred to as heartburn or regurgitation) and belching. Even very small, unnoticeable amounts of rising stomach acid may cause patients to wake up during the night.\n\nThis study will also investigate the effect of Rabeprazole, (brand name Aciphex) on patients with known insomnia. Rabeprazole is an FDA approved medication already marketed for the treatment of GERD.", "creators": [ { "firstName": "University of North Carolina", @@ -202,18 +235,19 @@ var testCases = [ } ], "date": "April 25, 2007", + "abstractNote": "This study will investigate Gastroesophageal Reflux Disease (GERD)as a cause of sleep disturbance. Patients with GERD may experience all or some of the following symptoms: stomach acid or partially digested food re-entering the esophagus (which is sometimes referred to as heartburn or regurgitation) and belching. Even very small, unnoticeable amounts of rising stomach acid may cause patients to wake up during the night.\n\nThis study will also investigate the effect of Rabeprazole, (brand name Aciphex) on patients with known insomnia. Rabeprazole is an FDA approved medication already marketed for the treatment of GERD.", "accessDate": "2020-04-01", - "libraryCatalog": "clinicaltrials.gov", - "shortTitle": "Sleep Disorders and Gastroesophageal Reflux Disease (GERD)", - "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", + "extra": "submitted: February 3, 2006", "institution": "clinicaltrials.gov", + "libraryCatalog": "clinicaltrials.gov", "reportNumber": "NCT00287391", "reportType": "Clinical trial registration", - "extra": "submitted: February 3, 2006", - "notes": [], + "shortTitle": "Sleep Disorders and Gastroesophageal Reflux Disease (GERD)", + "url": "https://clinicaltrials.gov/ct2/show/NCT00287391", + "attachments": [], "tags": [], - "seeAlso": [], - "attachments": [] + "notes": [], + "seeAlso": [] } ] }, @@ -224,7 +258,6 @@ var testCases = [ { "itemType": "report", "title": "Efficacy and Safety of Hydroxychloroquine for Treatment of COVID-19", - "abstractNote": "The study aims to evaluate the efficacy and safety of hydroxychloroquine in the treatment of COVID-19 pneumonia.", "creators": [ { "firstName": "Hongzhou", @@ -237,17 +270,18 @@ var testCases = [ } ], "date": "April 9, 2020", + "abstractNote": "The study aims to evaluate the efficacy and safety of hydroxychloroquine in the treatment of COVID-19 pneumonia.", "accessDate": "2020-04-01", - "libraryCatalog": "clinicaltrials.gov", - "url": "https://clinicaltrials.gov/ct2/show/NCT04261517", + "extra": "submitted: February 6, 2020", "institution": "clinicaltrials.gov", + "libraryCatalog": "clinicaltrials.gov", "reportNumber": "NCT04261517", "reportType": "Clinical trial registration", - "extra": "submitted: February 6, 2020", - "notes": [], + "url": "https://clinicaltrials.gov/ct2/show/NCT04261517", + "attachments": [], "tags": [], - "seeAlso": [], - "attachments": [] + "notes": [], + "seeAlso": [] } ] } From d70357b91e4478afe49b62633d18068dab692cea Mon Sep 17 00:00:00 2001 From: Sebastian Karcher Date: Mon, 4 May 2020 23:21:42 -0400 Subject: [PATCH 11/12] tweak detection --- clinicaltrials.gov.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 42f0b2c05e..2a19f49714 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -9,7 +9,7 @@ "inRepository": true, "translatorType": 4, "browserSupport": "gcsibv", - "lastUpdated": "2020-05-05 03:10:07" + "lastUpdated": "2020-05-05 03:20:08" } /* @@ -37,9 +37,11 @@ function detectWeb(doc, url) { - Zotero.monitorDOMChanges(doc.querySelector('#theDataTable')); - if (url.includes("/ct2/results?") && getSearchResults(doc, true)) { - return "multiple"; + if (url.includes("/ct2/results?")) { + Zotero.monitorDOMChanges(doc.querySelector('#theDataTable')); + if (getSearchResults(doc, true)) { + return "multiple"; + } } if (url.includes('/ct2/show')) { From e980cc12ad1ce6b56d9fe3933e9e6008fc4d3afc Mon Sep 17 00:00:00 2001 From: rdvelazquez Date: Tue, 5 May 2020 08:51:50 -0400 Subject: [PATCH 12/12] fix linter warnings with --- clinicaltrials.gov.js | 130 +++++++++++++++++++++++++----------------- 1 file changed, 77 insertions(+), 53 deletions(-) diff --git a/clinicaltrials.gov.js b/clinicaltrials.gov.js index 2a19f49714..cc8c540110 100644 --- a/clinicaltrials.gov.js +++ b/clinicaltrials.gov.js @@ -37,50 +37,51 @@ function detectWeb(doc, url) { - if (url.includes("/ct2/results?")) { - Zotero.monitorDOMChanges(doc.querySelector('#theDataTable')); - if (getSearchResults(doc, true)) { - return "multiple"; - } - } - - if (url.includes('/ct2/show')) { - return "report"; - } - - return false; + if (url.includes("/ct2/results?")) { + Zotero.monitorDOMChanges(doc.querySelector("#theDataTable")); + if (getSearchResults(doc, true)) { + return "multiple"; + } + } + + if (url.includes("/ct2/show")) { + return "report"; + } + + return false; } function getSearchResults(doc, checkOnly) { - var items = {}; - var found = false; - var rows = doc.querySelectorAll('table#theDataTable a[href*="/ct2/show"]'); - for (let row of rows) { - let href = row.href; - let title = ZU.trimInternal(row.textContent); - if (!href || !title) continue; - if (checkOnly) return true; - found = true; - items[href] = title; - } - return found ? items : false; + var items = {}; + var found = false; + var rows = doc.querySelectorAll('table#theDataTable a[href*="/ct2/show"]'); + for (let row of rows) { + let href = row.href; + let title = ZU.trimInternal(row.textContent); + if (!href || !title) continue; + if (checkOnly) return true; + found = true; + items[href] = title; + } + return found ? items : false; } function doWeb(doc, url) { - if (detectWeb(doc, url) == "multiple") { - Zotero.selectItems(getSearchResults(doc, false), function (items) { - if (items) ZU.processDocuments(Object.keys(items), scrape); - }); - } else - { - scrape(doc, url); - } + if (detectWeb(doc, url) == "multiple") { + Zotero.selectItems(getSearchResults(doc, false), function (items) { + if (items) ZU.processDocuments(Object.keys(items), scrape); + }); + } + else { + scrape(doc, url); + } } - - function isJsonAPIRequest(url) { - if (url.includes("https://clinicaltrials.gov/api/query") && url.includes("fmt=JSON")) { + if ( + url.includes("https://clinicaltrials.gov/api/query") + && url.includes("fmt=JSON") + ) { return true; } else { @@ -89,7 +90,10 @@ function isJsonAPIRequest(url) { } function isXmlAPIRequest(url) { - if (url.includes("https://clinicaltrials.gov/api/query") && url.includes("fmt=XML")) { + if ( + url.includes("https://clinicaltrials.gov/api/query") + && url.includes("fmt=XML") + ) { return true; } else { @@ -132,16 +136,32 @@ function scrape(doc, url) { let responsiblePartyInvestigator; let sponsor; let collaborators = []; - if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("ResponsibleParty")) { - const responsibleParty = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty; - if (typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string") { - responsiblePartyInvestigator = responsibleParty.ResponsiblePartyInvestigatorFullName; - creators.push(ZU.cleanAuthor(responsiblePartyInvestigator, "author", false)); + if ( + study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty( + "ResponsibleParty" + ) + ) { + const responsibleParty + = study.ProtocolSection.SponsorCollaboratorsModule.ResponsibleParty; + if ( + typeof responsibleParty.ResponsiblePartyInvestigatorFullName == "string" + ) { + responsiblePartyInvestigator + = responsibleParty.ResponsiblePartyInvestigatorFullName; + creators.push( + ZU.cleanAuthor(responsiblePartyInvestigator, "author", false) + ); } } - if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("LeadSponsor")) { - sponsor = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor.LeadSponsorName; + if ( + study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty( + "LeadSponsor" + ) + ) { + sponsor + = study.ProtocolSection.SponsorCollaboratorsModule.LeadSponsor + .LeadSponsorName; let sponsorCreatorType; if (creators.length == 0) { sponsorCreatorType = "author"; @@ -151,19 +171,23 @@ function scrape(doc, url) { } creators.push({ firstName: sponsor, - creatorType: sponsorCreatorType + creatorType: sponsorCreatorType, }); } - - if (study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty("CollaboratorList")) { - const collaboratorList = study.ProtocolSection.SponsorCollaboratorsModule.CollaboratorList.Collaborator; + + if ( + study.ProtocolSection.SponsorCollaboratorsModule.hasOwnProperty( + "CollaboratorList" + ) + ) { + const collaboratorList + = study.ProtocolSection.SponsorCollaboratorsModule.CollaboratorList + .Collaborator; collaboratorList.forEach((collaborator) => { - collaborators.push( - { - firstName: collaborator.CollaboratorName, - creatorType: "contributor" - } - ); + collaborators.push({ + firstName: collaborator.CollaboratorName, + creatorType: "contributor", + }); }); collaborators.forEach((collaborator) => { creators.push(collaborator);