Skip to content

Commit

Permalink
fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@jaanisoe
jaanisoe committed Dec 19, 2019
1 parent db7656d commit 0447fc8
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 17 deletions.
7 changes: 3 additions & 4 deletions cli/src/main/java/org/edamontology/pubfetcher/cli/DbFetch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,9 @@ public void run() {
}

logger.info((exceptionIndex ? "Refetch" : "Fetch") + " {} {}", databaseEntryType, PubFetcher.progress(localIndex + 1, databaseEntryIds.size(), progressStart));
if (stderr) {
System.err.print((exceptionIndex ? "Refetch" : "Fetch") + " " + databaseEntryType + " " + PubFetcher.progress(localIndex + 1, databaseEntryIds.size(), progressStart) + " \r");
}

DatabaseEntry<?> databaseEntry = null;
switch (databaseEntryType) {
Expand All @@ -146,10 +149,6 @@ public void run() {
case doc: databaseEntry = PubFetcher.getDoc((String) id, db, fetcher, fetcherArgs); break;
}

if (stderr) {
System.err.print((exceptionIndex ? "Refetch" : "Fetch") + " " + databaseEntryType + " " + PubFetcher.progress(localIndex + 1, databaseEntryIds.size(), progressStart) + " \r");
}

if (databaseEntry != null) {
if (databaseEntry.isFetchException() && !exceptionIndex) {
synchronized(exceptionIndexes) {
Expand Down
18 changes: 9 additions & 9 deletions cli/src/main/java/org/edamontology/pubfetcher/cli/PubFetcherMethods.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ private static void checkPublicationId(String publicationId) {
System.out.println(PubFetcher.getPublicationIds(publicationId, PUB_ID_SOURCE, true).toString(true));
}
private static void checkPublicationIds(String pmid, String pmcid, String doi) throws IllegalRequestException {
System.out.println(PubFetcher.getPublicationIds(pmid, pmcid, doi, PUB_ID_SOURCE, true, true).toString(true));
System.out.println(PubFetcher.getPublicationIds(pmid, pmcid, doi, PUB_ID_SOURCE, PUB_ID_SOURCE, PUB_ID_SOURCE, true, true).toString(true));
}
private static void checkUrl(String url) throws IllegalRequestException {
System.out.println(PubFetcher.getUrl(url, true));
Expand Down Expand Up @@ -267,7 +267,7 @@ private static List<? extends Object> idsCheck(List<? extends Object> ids, Datab
id = PubFetcher.getPublicationIds((String) s, PUB_ID_SOURCE, false);
} else {
PublicationIds pubId = (PublicationIds) s;
id = PubFetcher.getPublicationIds(pubId.getPmid(), pubId.getPmcid(), pubId.getDoi(), PUB_ID_SOURCE, false, true);
id = PubFetcher.getPublicationIds(pubId.getPmid(), pubId.getPmcid(), pubId.getDoi(), pubId.getPmidUrl(), pubId.getPmcidUrl(), pubId.getDoiUrl(), false, true);
}
break;
case webpage: case doc:
Expand Down Expand Up @@ -730,10 +730,10 @@ private static <T extends DatabaseEntry<T>> List<T> fetch(PubFetcherArgs args, S
long start = System.currentTimeMillis();
for (Object id : ids) {
logger.info("Fetch {} {}", type, PubFetcher.progress(i + 1, ids.size(), start));
T entry = fetchDatabaseEntry(id, null, fetcher, parts, fetcherArgs, false, type);
if (stderr) {
System.err.print("Fetch " + type + " " + PubFetcher.progress(i + 1, ids.size(), start) + " \r");
}
T entry = fetchDatabaseEntry(id, null, fetcher, parts, fetcherArgs, false, type);
if (entry != null) {
if (entry.isFetchException()) {
exceptionIndexes.add(i);
Expand All @@ -756,10 +756,10 @@ private static <T extends DatabaseEntry<T>> List<T> fetch(PubFetcherArgs args, S
for (int j = 0; j < exceptionIndexes.size(); ++j) {
i = exceptionIndexes.get(j);
logger.info("Refetch {} {}", type, PubFetcher.progress(i + 1, ids.size(), start));
T entry = fetchDatabaseEntry(exceptionIds.get(j), null, fetcher, parts, fetcherArgs, false, type);
if (stderr) {
System.err.print("Refetch " + type + " " + PubFetcher.progress(i + 1, ids.size(), start) + " \r");
}
T entry = fetchDatabaseEntry(exceptionIds.get(j), null, fetcher, parts, fetcherArgs, false, type);
if (entry != null) {
if (preFilter(args, fetcher, fetcherArgs, entry, type)) {
entries.set(i, entry);
Expand Down Expand Up @@ -794,10 +794,10 @@ private static <T extends DatabaseEntry<T>> List<T> fetchPut(PubFetcherArgs args
long start = System.currentTimeMillis();
for (Object id : ids) {
logger.info("Fetch {} {}", type, PubFetcher.progress(i + 1, ids.size(), start));
T entry = fetchDatabaseEntry(id, db, fetcher, parts, fetcherArgs, true, type);
if (stderr) {
System.err.print("Fetch " + type + " " + PubFetcher.progress(i + 1, ids.size(), start) + " \r");
}
T entry = fetchDatabaseEntry(id, db, fetcher, parts, fetcherArgs, true, type);
if (entry != null) {
if (entry.isFetchException()) {
exceptionIndexes.add(i);
Expand All @@ -820,10 +820,10 @@ private static <T extends DatabaseEntry<T>> List<T> fetchPut(PubFetcherArgs args
for (int j = 0; j < exceptionIndexes.size(); ++j) {
i = exceptionIndexes.get(j);
logger.info("Refetch {} {}", type, PubFetcher.progress(i + 1, ids.size(), start));
T entry = fetchDatabaseEntry(exceptionIds.get(j), db, fetcher, parts, fetcherArgs, true, type);
if (stderr) {
System.err.print("Refetch " + type + " " + PubFetcher.progress(i + 1, ids.size(), start) + " \r");
}
T entry = fetchDatabaseEntry(exceptionIds.get(j), db, fetcher, parts, fetcherArgs, true, type);
if (entry != null) {
if (preFilter(args, fetcher, fetcherArgs, entry, type)) {
entries.set(i, entry);
Expand Down Expand Up @@ -986,15 +986,15 @@ private static void updateCitationsCount(List<Publication> publications, String
for (int i = 0; i < publications.size(); ++i) {
Publication publication = publications.get(i);
logger.info("Update citations count {}", PubFetcher.progress(i + 1, publications.size(), start));
if (stderr) {
System.err.print("Update citations count " + PubFetcher.progress(i + 1, publications.size(), start) + " \r");
}
if (fetcher.updateCitationsCount(publication, fetcherArgs)) {
db.putPublication(publication);
db.commit();
} else {
++fail;
}
if (stderr) {
System.err.print("Update citations count " + PubFetcher.progress(i + 1, publications.size(), start) + " \r");
}
}
}
if (fail > 0) logger.warn("Failed to update citations count of {} publications", fail);
Expand Down
8 changes: 5 additions & 3 deletions core/src/main/java/org/edamontology/pubfetcher/core/common/PubFetcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,15 +415,17 @@ public static PublicationIds getPublicationIds(String publicationId, String url,
* @param pmid the string representing the PMID of the publication
* @param pmcid the string representing the PMCID of the publication
* @param doi the string representing the DOI of the publication
* @param url the string of the URL where the given PMID, PMCID and DOI were found
* @param pmidUrl the string of the URL where the given PMID was found
* @param pmcidUrl the string of the URL where the given PMCID was found
* @param doiUrl the string of the URL where the given DOI was found
* @param throwException if <code>true</code> then an {@link IllegalRequestException} will be thrown if any of the given non-empty PMID, PMCID or DOI is
* not valid or no valid IDs are given; if <code>false</code> then only error messages will be logged for invalid IDs
* @param logEmpty if <code>true</code> then an error message will be logged if no valid IDs are given
* @return a {@link PublicationIds} for the given PMID, PMCID and DOI string and ID provenance URL; <code>null</code> if no valid IDs are given
* @throws IllegalRequestException if any of the given non-empty PMID, PMCID or DOI is not valid or no valid IDs are given, and
* <code>throwException</code> is <code>true</code>
*/
public static PublicationIds getPublicationIds(String pmid, String pmcid, String doi, String url, boolean throwException, boolean logEmpty) throws IllegalRequestException {
public static PublicationIds getPublicationIds(String pmid, String pmcid, String doi, String pmidUrl, String pmcidUrl, String doiUrl, boolean throwException, boolean logEmpty) throws IllegalRequestException {
if (pmid == null || pmid.trim().isEmpty()) pmid = null;
else if (!isPmid(pmid)) {
logger.error("Invalid PMID: {}", pmid);
Expand Down Expand Up @@ -458,7 +460,7 @@ else if (!isDoi(doi)) {
return null;
}
return new PublicationIds(pmid, pmcid, doi,
pmid == null ? null : url, pmcid == null ? null : url, doi == null ? null : url);
pmid == null ? null : pmidUrl, pmcid == null ? null : pmcidUrl, doi == null ? null : doiUrl);
}

/**
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/edamontology/pubfetcher/core/fetching/FetcherTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public final class FetcherTest {

private static final Logger logger = LogManager.getLogger();

private static String PUB_ID_SOURCE = "PubFetcher";
private static String PUB_ID_SOURCE = "PubFetcherTest";

private static InputStream getResource(String name, String prefix) {
InputStream resource = FetcherTest.class.getResourceAsStream("/" + prefix + "/" + name);
Expand Down

0 comments on commit 0447fc8

Please sign in to comment.