diff --git a/bull/pdl-queue/consumer.js b/bull/pdl-queue/consumer.js
index c1417d1..450a98a 100644
--- a/bull/pdl-queue/consumer.js
+++ b/bull/pdl-queue/consumer.js
@@ -28,33 +28,42 @@ async function getZipAndBytelength(no_of_pages, id, title, job) {
title = title.replace(/ /g, "_");
var img = zip.folder(`${title}_images`);
let temp_pages = no_of_pages;
+ let downloadImageStatus;
+ let errorFlag = { status: false, page: "" };
var download_image = async function (uri, filename) {
- await rp({
- method: "GET",
- uri,
- encoding: null,
- transform: function (body, response) {
- return { headers: response.headers, data: body };
- },
- })
- .then(async function (body) {
- if (/image/.test(body.headers["content-type"])) {
- var data = new Buffer(body.data);
- img.file(filename, data.toString("base64"), { base64: true });
- }
- })
- .catch(function (err) {
- --no_of_pages;
+ try {
+ const body = await rp({
+ method: "GET",
+ uri,
+ encoding: null,
+ transform: function (body, response) {
+ return { headers: response.headers, data: body };
+ },
});
+ if (/image/.test(body.headers["content-type"])) {
+ var data = new Buffer(body.data);
+ img.file(filename, data.toString("base64"), { base64: true });
+ }
+ return 200;
+ } catch (err) {
+ --no_of_pages;
+ return err.statusCode;
+ }
};
for (let i = 1; i <= temp_pages; ++i) {
const str = `http://www.panjabdigilib.org/images?ID=${id}&page=${i}&pagetype=null&Searched=W3GX`;
- await download_image(str, `${title}_${i}.jpeg`);
+ downloadImageStatus = await download_image(str, `${title}_${i}.jpeg`);
job.progress(Math.round((i / temp_pages) * 82));
+ if (downloadImageStatus >= 200 && downloadImageStatus < 300) {
+ continue;
+ } else {
+ errorFlag = { status: true, page: str };
+ break;
+ }
}
let { byteLength } = await zip.generateAsync({ type: "nodebuffer" });
byteLength = Number(byteLength + no_of_pages * 16); //No. of pages * 16
- return [zip, byteLength];
+ return [zip, byteLength, errorFlag];
}
function setHeaders(metadata, byteLength, title) {
@@ -117,22 +126,37 @@ PDLQueue.process(async (job, done) => {
const trueURI = `http://archive.org/details/${job.data.details.IAIdentifier}`;
jobLogs["trueURI"] = trueURI;
jobLogs["userName"] = job.data.details.userName;
- job.log(JSON.stringify(jobLogs));
- logUserData(jobLogs["userName"], "Panjab Digital Library");
- const [zip, byteLength] = await getZipAndBytelength(
+ const [zip, byteLength, errorFlag] = await getZipAndBytelength(
job.data.details.Pages,
job.data.details.bookID,
job.data.details.title,
job
);
- job.progress(90);
- await uploadToIA(
- zip,
- job.data.details,
- byteLength,
- job.data.details.email,
- job
- );
- job.progress(100);
- done(null, true);
+ if (errorFlag.status) {
+ job.log(JSON.stringify(jobLogs));
+ logUserData(jobLogs["userName"], "Panjab Digital Library");
+ logger.log({
+ level: "error",
+ message: `Upload to Internet Archive failed because ${errorFlag.page} is not reachable. Please try again or contact Panjab Digital Library for more details.`,
+ });
+ job.progress(100);
+ done(
+ new Error(
+ `Upload to Internet Archive failed because ${errorFlag.page} is not reachable. Please try again or contact Panjab Digital Library for more details.`
+ )
+ );
+ } else {
+ job.log(JSON.stringify(jobLogs));
+ logUserData(jobLogs["userName"], "Panjab Digital Library");
+ job.progress(90);
+ await uploadToIA(
+ zip,
+ job.data.details,
+ byteLength,
+ job.data.details.email,
+ job
+ );
+ job.progress(100);
+ done(null, true);
+ }
});
diff --git a/components/QueueTable.js b/components/QueueTable.js
index bc4cf30..f99220a 100644
--- a/components/QueueTable.js
+++ b/components/QueueTable.js
@@ -121,6 +121,22 @@ const ShowUploadQueue = (props) => {
label: "Status",
minWidth: 30,
align: "left",
+ format: (value) => {
+ const isPDLMissingPage = /]*>([^<]+)<\/a>/;
+ const missingPageLink = isPDLMissingPage.exec(value);
+ return missingPageLink ? (
+
+ Failed! (Reason: Upload to Internet Archive failed because {""}
+
+ {missingPageLink[1]}
+ {" "}
+ is not reachable. Please try again or contact Panjab Digital Library
+ for more details. )
+
+ ) : (
+ value
+ );
+ },
},
{
id: "upload_progress",
@@ -158,6 +174,8 @@ const ShowUploadQueue = (props) => {
return column.format((value === "-" ? "" : "User:") + value);
} else if (column.id === "date") {
return column.format(value);
+ } else if (column.id === "status") {
+ return column.format(value);
} else {
return value;
}