diff --git a/bull/pdl-queue/consumer.js b/bull/pdl-queue/consumer.js index c1417d1..450a98a 100644 --- a/bull/pdl-queue/consumer.js +++ b/bull/pdl-queue/consumer.js @@ -28,33 +28,42 @@ async function getZipAndBytelength(no_of_pages, id, title, job) { title = title.replace(/ /g, "_"); var img = zip.folder(`${title}_images`); let temp_pages = no_of_pages; + let downloadImageStatus; + let errorFlag = { status: false, page: "" }; var download_image = async function (uri, filename) { - await rp({ - method: "GET", - uri, - encoding: null, - transform: function (body, response) { - return { headers: response.headers, data: body }; - }, - }) - .then(async function (body) { - if (/image/.test(body.headers["content-type"])) { - var data = new Buffer(body.data); - img.file(filename, data.toString("base64"), { base64: true }); - } - }) - .catch(function (err) { - --no_of_pages; + try { + const body = await rp({ + method: "GET", + uri, + encoding: null, + transform: function (body, response) { + return { headers: response.headers, data: body }; + }, }); + if (/image/.test(body.headers["content-type"])) { + var data = new Buffer(body.data); + img.file(filename, data.toString("base64"), { base64: true }); + } + return 200; + } catch (err) { + --no_of_pages; + return err.statusCode; + } }; for (let i = 1; i <= temp_pages; ++i) { const str = `http://www.panjabdigilib.org/images?ID=${id}&page=${i}&pagetype=null&Searched=W3GX`; - await download_image(str, `${title}_${i}.jpeg`); + downloadImageStatus = await download_image(str, `${title}_${i}.jpeg`); job.progress(Math.round((i / temp_pages) * 82)); + if (downloadImageStatus >= 200 && downloadImageStatus < 300) { + continue; + } else { + errorFlag = { status: true, page: str }; + break; + } } let { byteLength } = await zip.generateAsync({ type: "nodebuffer" }); byteLength = Number(byteLength + no_of_pages * 16); //No. of pages * 16 - return [zip, byteLength]; + return [zip, byteLength, errorFlag]; } function setHeaders(metadata, byteLength, title) { @@ -117,22 +126,37 @@ PDLQueue.process(async (job, done) => { const trueURI = `http://archive.org/details/${job.data.details.IAIdentifier}`; jobLogs["trueURI"] = trueURI; jobLogs["userName"] = job.data.details.userName; - job.log(JSON.stringify(jobLogs)); - logUserData(jobLogs["userName"], "Panjab Digital Library"); - const [zip, byteLength] = await getZipAndBytelength( + const [zip, byteLength, errorFlag] = await getZipAndBytelength( job.data.details.Pages, job.data.details.bookID, job.data.details.title, job ); - job.progress(90); - await uploadToIA( - zip, - job.data.details, - byteLength, - job.data.details.email, - job - ); - job.progress(100); - done(null, true); + if (errorFlag.status) { + job.log(JSON.stringify(jobLogs)); + logUserData(jobLogs["userName"], "Panjab Digital Library"); + logger.log({ + level: "error", + message: `Upload to Internet Archive failed because ${errorFlag.page} is not reachable. Please try again or contact Panjab Digital Library for more details.`, + }); + job.progress(100); + done( + new Error( + `Upload to Internet Archive failed because ${errorFlag.page} is not reachable. Please try again or contact Panjab Digital Library for more details.` + ) + ); + } else { + job.log(JSON.stringify(jobLogs)); + logUserData(jobLogs["userName"], "Panjab Digital Library"); + job.progress(90); + await uploadToIA( + zip, + job.data.details, + byteLength, + job.data.details.email, + job + ); + job.progress(100); + done(null, true); + } }); diff --git a/components/QueueTable.js b/components/QueueTable.js index bc4cf30..f99220a 100644 --- a/components/QueueTable.js +++ b/components/QueueTable.js @@ -121,6 +121,22 @@ const ShowUploadQueue = (props) => { label: "Status", minWidth: 30, align: "left", + format: (value) => { + const isPDLMissingPage = /]*>([^<]+)<\/a>/; + const missingPageLink = isPDLMissingPage.exec(value); + return missingPageLink ? ( + + Failed! (Reason: Upload to Internet Archive failed because {""} + + {missingPageLink[1]} + {" "} + is not reachable. Please try again or contact Panjab Digital Library + for more details. ) + + ) : ( + value + ); + }, }, { id: "upload_progress", @@ -158,6 +174,8 @@ const ShowUploadQueue = (props) => { return column.format((value === "-" ? "" : "User:") + value); } else if (column.id === "date") { return column.format(value); + } else if (column.id === "status") { + return column.format(value); } else { return value; }