From 26658b666517f9da71de5e1ff706dce692da6350 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Thu, 16 May 2024 16:36:35 +0200 Subject: [PATCH 1/4] evaluate surname and given-name if authname is not present (cherry picked from commit 8b5f0445373e5ce60be2b12fd1d487f54f74e350) --- .../contributor/AuthorMetadataContributor.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/AuthorMetadataContributor.java b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/AuthorMetadataContributor.java index 26063dc7441..62b7fe81e39 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/AuthorMetadataContributor.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/metadatamapping/contributor/AuthorMetadataContributor.java @@ -71,7 +71,7 @@ public Collection contributeMetadata(Element element) { } /** - * Retrieve the the ScopusID, orcid, author name and affiliationID + * Retrieve the ScopusID, orcid, author name and affiliationID * metadata associated with the given element object. * If the value retrieved from the element is empty * it is set PLACEHOLDER_PARENT_METADATA_VALUE @@ -82,11 +82,19 @@ public Collection contributeMetadata(Element element) { private List getMetadataOfAuthors(Element element) throws JaxenException { List metadatums = new ArrayList(); Element authname = element.getChild("authname", NAMESPACE); + Element surname = element.getChild("surname", NAMESPACE); + Element givenName = element.getChild("given-name", NAMESPACE); Element scopusId = element.getChild("authid", NAMESPACE); Element orcid = element.getChild("orcid", NAMESPACE); Element afid = element.getChild("afid", NAMESPACE); - addMetadatum(metadatums, getMetadata(getElementValue(authname), this.authname)); + if (authname != null) { + addMetadatum(metadatums, getMetadata(getElementValue(authname), this.authname)); + } else { + addMetadatum(metadatums, getMetadata(getElementValue(surname) + ", " + + getElementValue(givenName), this.authname)); + } + addMetadatum(metadatums, getMetadata(getElementValue(scopusId), this.scopusId)); addMetadatum(metadatums, getMetadata(getElementValue(orcid), this.orcid)); addMetadatum(metadatums, getMetadata(StringUtils.isNotBlank(afid.getValue()) @@ -170,4 +178,4 @@ public void setAffiliation(MetadataFieldConfig affiliation) { this.affiliation = affiliation; } -} \ No newline at end of file +} From 6e5c5089abe776b0e3b8b364ec10393855275429 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Thu, 16 May 2024 16:37:48 +0200 Subject: [PATCH 2/4] Update scopus-empty-resp.xml (cherry picked from commit 4b2ea66f19324d204058084434ff99a71fd6588e) --- .../resources/org/dspace/app/rest/scopus-empty-resp.xml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/scopus-empty-resp.xml b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/scopus-empty-resp.xml index b2b4264b5c3..3e0ecbead35 100644 --- a/dspace-server-webapp/src/test/resources/org/dspace/app/rest/scopus-empty-resp.xml +++ b/dspace-server-webapp/src/test/resources/org/dspace/app/rest/scopus-empty-resp.xml @@ -3,9 +3,11 @@ 0 0 0 - - + + Result set was empty - \ No newline at end of file + From 2b1270197e2d4af1d7051a2d75c501d00d3fc318 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Thu, 16 May 2024 16:40:04 +0200 Subject: [PATCH 3/4] improve handling of 0 hits responses of Scopus API (cherry picked from commit b12bd6ce565bb95fcb470a4ea156abd0a6ca531f) --- ...ScopusImportMetadataSourceServiceImpl.java | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/ScopusImportMetadataSourceServiceImpl.java b/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/ScopusImportMetadataSourceServiceImpl.java index 944d467e315..e61ca052868 100644 --- a/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/ScopusImportMetadataSourceServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/importer/external/scopus/service/ScopusImportMetadataSourceServiceImpl.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -24,6 +25,8 @@ import javax.el.MethodNotFoundException; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.importer.external.datamodel.ImportRecord; import org.dspace.importer.external.datamodel.Query; @@ -62,6 +65,8 @@ public class ScopusImportMetadataSourceServiceImpl extends AbstractImportMetadat @Autowired private LiveImportClient liveImportClient; + private final static Logger log = LogManager.getLogger(); + public LiveImportClient getLiveImportClient() { return liveImportClient; } @@ -200,6 +205,9 @@ public Integer call() throws Exception { Map requestParams = getRequestParameters(query, null, null, null); params.put(URI_PARAMETERS, requestParams); String response = liveImportClient.executeHttpGetRequest(timeout, url, params); + if (StringUtils.isEmpty(response)) { + return 0; + } SAXBuilder saxBuilder = new SAXBuilder(); // disallow DTD parsing to ensure no XXE attacks can occur @@ -245,6 +253,10 @@ public List call() throws Exception { Map requestParams = getRequestParameters(queryString, viewMode, null, null); params.put(URI_PARAMETERS, requestParams); String response = liveImportClient.executeHttpGetRequest(timeout, url, params); + if (StringUtils.isEmpty(response)) { + return results; + } + List elements = splitToRecords(response); for (Element record : elements) { results.add(transformSourceRecords(record)); @@ -304,6 +316,10 @@ public List call() throws Exception { Map requestParams = getRequestParameters(queryString, viewMode, start, count); params.put(URI_PARAMETERS, requestParams); String response = liveImportClient.executeHttpGetRequest(timeout, url, params); + if (StringUtils.isEmpty(response)) { + return results; + } + List elements = splitToRecords(response); for (Element record : elements) { results.add(transformSourceRecords(record)); @@ -349,6 +365,10 @@ public List call() throws Exception { Map requestParams = getRequestParameters(queryString, viewMode, start, count); params.put(URI_PARAMETERS, requestParams); String response = liveImportClient.executeHttpGetRequest(timeout, url, params); + if (StringUtils.isEmpty(response)) { + return results; + } + List elements = splitToRecords(response); for (Element record : elements) { results.add(transformSourceRecords(record)); @@ -383,10 +403,16 @@ private List splitToRecords(String recordsSrc) { saxBuilder.setFeature("http://apache.org/xml/features/disallow-doctype-decl",true); Document document = saxBuilder.build(new StringReader(recordsSrc)); Element root = document.getRootElement(); - List records = root.getChildren("entry",Namespace.getNamespace("http://www.w3.org/2005/Atom")); + String totalResults = root.getChildText("totalResults", Namespace.getNamespace("http://a9.com/-/spec/opensearch/1.1/")); + if (totalResults != null && "0".equals(totalResults)) { + log.debug("got Scopus API with empty response"); + return Collections.emptyList(); + } + List records = root.getChildren("entry", Namespace.getNamespace("http://www.w3.org/2005/Atom")); return records; } catch (JDOMException | IOException e) { - return new ArrayList(); + log.warn("got unexpected XML response from Scopus API: " + e.getMessage()); + return Collections.emptyList(); } } @@ -422,4 +448,4 @@ public void setInstKey(String instKey) { this.instKey = instKey; } -} \ No newline at end of file +} From 8e56fdd95b2054a49606103554f5bea1366fce21 Mon Sep 17 00:00:00 2001 From: Sascha Szott Date: Thu, 16 May 2024 16:41:31 +0200 Subject: [PATCH 4/4] fixed failed test (cherry picked from commit 6989cb6f15761ac453ada2b5ce69b6d2dbbd3e78) --- .../app/rest/ScopusImportMetadataSourceServiceIT.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java index b9310e3adba..7f6cb53ce40 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ScopusImportMetadataSourceServiceIT.java @@ -96,7 +96,7 @@ public void scopusImportMetadataGetRecordsCountTest() throws Exception { } @Test - public void scopusImportMetadataGetRecordsEmptyResponceTest() throws Exception { + public void scopusImportMetadataGetRecordsEmptyResponseTest() throws Exception { context.turnOffAuthorisationSystem(); String originApiKey = scopusServiceImpl.getApiKey(); if (StringUtils.isBlank(originApiKey)) { @@ -113,8 +113,7 @@ public void scopusImportMetadataGetRecordsEmptyResponceTest() throws Exception { context.restoreAuthSystemState(); Collection recordsImported = scopusServiceImpl.getRecords("roma", 0, 20); - ImportRecord importedRecord = recordsImported.iterator().next(); - assertTrue(importedRecord.getValueList().isEmpty()); + assertTrue(recordsImported.isEmpty()); } finally { liveImportClientImpl.setHttpClient(originalHttpClient); scopusServiceImpl.setApiKey(originApiKey); @@ -229,4 +228,4 @@ private ArrayList getRecords() { return records; } -} \ No newline at end of file +}