diff --git a/bull/commons-queue/consumer.js b/bull/commons-queue/consumer.js index 79cd8e4..bb6d84b 100644 --- a/bull/commons-queue/consumer.js +++ b/bull/commons-queue/consumer.js @@ -1,8 +1,12 @@ const config = require("../../utils/bullconfig"); const CommonsQueue = config.getNewQueue("commons-queue"); const winston = require("winston"); -const { downloadFile, uploadToCommons } = require("../../utils/helper"); -const logger = winston.loggers.get("defaultLogger"); +const { + downloadFile, + uploadToCommons, + uploadToWikiData, + uploadToWikiSource, +} = require("../../utils/helper"); CommonsQueue.process(async (job, done) => { const downloadFileRes = await downloadFile( @@ -11,34 +15,59 @@ CommonsQueue.process(async (job, done) => { ); if (downloadFileRes.writeFileStatus !== 200) { - logger.log({ - level: "error", - message: `downloadFile: ${downloadFileRes}`, - }); process.emit(`commonsJobComplete:${job.id}`, { status: false, value: null, }); return done(null, true); - // return done(new Error(`downloadFile: ${downloadFileRes}`)); } const commonsResponse = await uploadToCommons(job.data.metadata); if (commonsResponse.fileUploadStatus !== 200) { - logger.log({ - level: "error", - message: `uploadToCommons: ${commonsResponse}`, - }); process.emit(`commonsJobComplete:${job.id}`, { status: false, value: null, }); return done(null, true); - // return done(new Error(`uploadToCommons: ${commonsResponse}`)); } - process.emit(`commonsJobComplete:${job.id}`, { - status: true, - value: commonsResponse, - }); + const wikiDataResponse = await uploadToWikiData( + job.data.metadata, + commonsResponse.filename + ); + if (wikiDataResponse !== 404) { + const wikisourceResponse = await uploadToWikiSource( + job.data.metadata, + wikiDataResponse + ); + + if (wikisourceResponse.fileUploadStatus === 200) { + process.emit(`commonsJobComplete:${job.id}`, { + status: true, + value: { + commons: commonsResponse, + wikidata: wikiDataResponse, + wikisource: wikisourceResponse, + }, + }); + } else { + process.emit(`commonsJobComplete:${job.id}`, { + status: true, + value: { + commons: commonsResponse, + wikidata: wikiDataResponse, + wikisource: 404, + }, + }); + } + } else { + process.emit(`commonsJobComplete:${job.id}`, { + status: true, + value: { + commons: commonsResponse, + wikidata: 404, + wikisource: 404, + }, + }); + } return done(null, true); }); diff --git a/bull/google-books-queue/consumer.js b/bull/google-books-queue/consumer.js index 86783c3..dd8cf52 100644 --- a/bull/google-books-queue/consumer.js +++ b/bull/google-books-queue/consumer.js @@ -125,7 +125,15 @@ GoogleBooksQueue.process((job, done) => { step: "Upload to Wikimedia Commons", value: `(${100}%)`, wikiLinks: { - commons: await commonsResponse.value.filename, + commons: await commonsResponse.value.commons.filename, + wikidata: + (await commonsResponse.value.wikidata) !== 404 + ? await commonsResponse.value.wikidata + : 404, + wikisource: + (await commonsResponse.value.wikisource) !== 404 + ? await commonsResponse.value.wikisource.filename + : 404, }, }); if (job.data.isEmailNotification === "true") { diff --git a/bull/trove-queue/consumer.js b/bull/trove-queue/consumer.js index b509715..ef9cd85 100644 --- a/bull/trove-queue/consumer.js +++ b/bull/trove-queue/consumer.js @@ -107,6 +107,12 @@ TroveQueue.process((job, done) => { step: "Upload To IA", value: `(${100}%)`, }); + if ( + isEmailNotification !== "true" && + job.data.details.isUploadCommons !== "true" + ) { + done(null, true); + } if ( isEmailNotification === "true" && job.data.details.isUploadCommons !== "true" @@ -128,9 +134,19 @@ TroveQueue.process((job, done) => { if (commonsResponse.status === true) { job.progress({ step: "Upload to Wikimedia Commons", - value: `(100%)`, + value: `(${100}%)`, wikiLinks: { - commons: await commonsResponse.value.filename, + commons: await commonsResponse.value.commons + .filename, + wikidata: + (await commonsResponse.value.wikidata) !== 404 + ? await commonsResponse.value.wikidata + : 404, + wikisource: + (await commonsResponse.value.wikisource) !== 404 + ? await commonsResponse.value.wikisource + .filename + : 404, }, }); if (job.data.isEmailNotification === "true") { @@ -138,7 +154,10 @@ TroveQueue.process((job, done) => { EmailProducer( userName, name, - { archiveLink: trueURI, commonsLink: commonsLink }, + { + archiveLink: trueURI, + commonsLink: commonsLink, + }, { archive: true, commons: true } ); } diff --git a/components/ShowJobInformation.js b/components/ShowJobInformation.js index b85aa2a..01d247c 100644 --- a/components/ShowJobInformation.js +++ b/components/ShowJobInformation.js @@ -18,7 +18,8 @@ const ShowJobInformation = (props) => { const styles = { root: { maxWidth: 365, - height: "fit-content", + height: "95vh", + overflow: "scroll", }, cardContentContainer: { height: "200px", @@ -144,10 +145,10 @@ const ShowJobInformation = (props) => { - {data.wikimedia_links !== "Not Integrated" ? ( + {data.wikimedia_links.commons !== "Not Integrated" ? ( + + ) : null} + + + + {data.wikimedia_links.wikisource !== "Not Integrated" ? ( + + + + ) : null} + ); diff --git a/server.js b/server.js index 39091a6..c95c0b3 100644 --- a/server.js +++ b/server.js @@ -176,9 +176,21 @@ app : "", isUploaded: jobState === "completed" ? true : false, }, - wikimedia_links: job.progress().wikiLinks?.commons - ? job.progress().wikiLinks.commons - : "Not Integrated", + wikimedia_links: { + commons: job.progress().wikiLinks?.commons + ? job.progress().wikiLinks.commons + : "Not Integrated", + wikidata: + job.progress().wikiLinks?.wikidata && + job.progress().wikiLinks.wikidata !== 404 + ? job.progress().wikiLinks.wikidata + : "Not Integrated", + wikisource: + job.progress().wikiLinks?.wikisource && + job.progress().wikiLinks.wikisource !== 404 + ? job.progress().wikiLinks.wikisource + : "Not Integrated", + }, }; res.send( Object.assign( diff --git a/utils/helper.js b/utils/helper.js index 7707b9b..c8c958d 100644 --- a/utils/helper.js +++ b/utils/helper.js @@ -295,12 +295,289 @@ module.exports = { title, metadata.commonsMetadata ); + logger.log({ + level: "info", + message: `uploadToCommons: Upload of ${metadata.IAIdentifier} to commons successful`, + }); + return { + fileUploadStatus: 200, + filename: response.filename, + }; + } catch (error) { + await fs.promises.unlink("commonsFilePayload.pdf"); + logger.log({ + level: "error", + message: `uploadToCommons: ${error}`, + }); + return error; + } + }, + uploadToWikiData: async (metadata, commonsItemFilename) => { + const bot = await Mwn.init({ + apiUrl: "https://www.wikidata.org/w/api.php", + OAuth2AccessToken: metadata.oauthToken, + userAgent: "bub2.toolforge ([[https://bub2.toolforge.org]])", + defaultParams: { + assert: "user", + }, + }); + + async function createEntity(csrf_token) { + try { + const title = metadata.details.volumeInfo.title || ""; + const id = metadata.details.id || ""; + const authorsArr = metadata.details.volumeInfo.authors + ? metadata.details.volumeInfo.authors.join().trim() + : null; + const authors = authorsArr || ""; + // Mapping for the labels/properties defined in `payload` - https://prop-explorer.toolforge.org/ + const payload = { + labels: { + en: { + language: "en", + value: commonsItemFilename, + }, + }, + descriptions: { + en: { + language: "en", + value: title, + }, + }, + claims: { + file_name: [ + { + mainsnak: { + snaktype: "value", + property: "P18", + datavalue: { + value: commonsItemFilename, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + file_url: [ + { + mainsnak: { + snaktype: "value", + property: "P4765", + datavalue: { + value: `https://commons.wikimedia.org/wiki/File:${commonsItemFilename}`, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + commons_category: [ + { + mainsnak: { + snaktype: "value", + property: "P373", + datavalue: { + value: "Bub.wikimedia", + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + internet_archive_id: id + ? [ + { + mainsnak: { + snaktype: "value", + property: "P724", + datavalue: { + value: id, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + collection: [ + { + mainsnak: { + snaktype: "value", + property: "P195", + datavalue: { + value: { + "entity-type": "item", + "numeric-id": 39162, + id: "Q39162", //wikidataID for 'opensource' + }, + type: "wikibase-entityid", + }, + }, + type: "statement", + rank: "normal", + }, + ], + title: title + ? [ + { + mainsnak: { + snaktype: "value", + property: "P1476", + datavalue: { + value: { + text: title, + language: "en", + }, + type: "monolingualtext", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + name: title + ? [ + { + mainsnak: { + snaktype: "value", + property: "P2561", + datavalue: { + value: { + text: title, + language: "en", + }, + type: "monolingualtext", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + file_format: [ + { + mainsnak: { + snaktype: "value", + property: "P2701", + datavalue: { + value: { + "entity-type": "item", + "numeric-id": 42332, + id: "Q42332", // wikidataID for PDF + }, + type: "wikibase-entityid", + }, + }, + type: "statement", + rank: "normal", + }, + ], + author_name: authors + ? [ + { + mainsnak: { + snaktype: "value", + property: "P2093", + datavalue: { + value: authors, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + URL: [ + { + mainsnak: { + snaktype: "value", + property: "P2699", + datavalue: { + value: `https://commons.wikimedia.org/wiki/File:${commonsItemFilename}`, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + copyright_status: [ + { + mainsnak: { + snaktype: "value", + property: "P6216", + datavalue: { + value: { + "entity-type": "item", + "numeric-id": 6938433, + id: "Q6938433", // wikidataID for CC0 license + }, + type: "wikibase-entityid", + }, + }, + type: "statement", + rank: "normal", + }, + ], + }, + }; + + const res = await bot.request({ + action: "wbeditentity", + new: "item", + summary: "bub2.toolforge.org: upload commons item to wikidata", + tags: "wikimedia-commons-app", + data: JSON.stringify(payload), + token: csrf_token, + }); + logger.log({ + level: "info", + message: `uploadToWikidata: Upload of ${commonsItemFilename} metadata to wikidata successful`, + }); + return res.entity.id; + } catch (error) { + await fs.promises.unlink("commonsFilePayload.pdf"); + logger.log({ + level: "error", + message: `uploadToWikidata:${error}`, + }); + return 404; + } + } + const csrf_token = await bot.getCsrfToken(); + return await createEntity(csrf_token); + }, + uploadToWikiSource: async (metadata, commonsItemFilename) => { + try { + const bot = await Mwn.init({ + apiUrl: "https://en.wikisource.org/w/api.php", + OAuth2AccessToken: metadata.oauthToken, + userAgent: "bub2.toolforge ([[https://bub2.toolforge.org]])", + defaultParams: { + assert: "user", + }, + }); + + const commonsFilePayload = "commonsFilePayload.pdf"; + const title = metadata.details.volumeInfo.title || metadata.name; + const response = await bot.upload( + commonsFilePayload, + `bub2_${title}`, + "" + ); + console.log("response:", response); if (await response.filename) { await fs.promises.unlink(commonsFilePayload); } logger.log({ level: "info", - message: `uploadToCommons: Upload of ${metadata.IAIdentifier} to commons successful`, + message: `uploadToWikiSource: Upload of ${commonsItemFilename} to wikisource successful`, }); return { fileUploadStatus: 200, @@ -310,9 +587,69 @@ module.exports = { await fs.promises.unlink("commonsFilePayload.pdf"); logger.log({ level: "error", - message: `uploadToCommons: ${error}`, + message: `uploadToWikiSource: ${error}`, }); return error; } }, + + updateWikiData: async (metadata, wikidataId, wikisourceIndex) => { + try { + const bot = await Mwn.init({ + apiUrl: "https://www.wikidata.org/w/api.php", + OAuth2AccessToken: metadata.oauthToken, + userAgent: "bub2.toolforge ([[https://bub2.toolforge.org]])", + defaultParams: { + assert: "user", + }, + }); + + const payload = { + claims: { + wikisource_index_page: [ + { + mainsnak: { + snaktype: "value", + property: "P1957", + datavalue: { + value: wikisourceIndex, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + }, + }; + + const csrf_token = await bot.getCsrfToken(); + const res = await bot.request({ + action: "wbeditentity", + id: wikidataId, + data: JSON.stringify(payload), + summary: "Update wikisource index page from bub2.toolforge.org", + token: csrf_token, + }); + + logger.log({ + level: "info", + message: `updateWikiData: Update of Wikidata item ${wikidataId} successful`, + }); + + return { + success: true, + entityId: wikidataId, + }; + } catch (error) { + logger.log({ + level: "error", + message: `updateWikiData: ${error}`, + }); + return { + success: false, + error: error, + }; + } + }, };