From 414677dea59c76577d07e62c776e0d1ffde73f9c Mon Sep 17 00:00:00 2001 From: Okereke Chinweotito Date: Sat, 27 Jan 2024 12:55:25 +0100 Subject: [PATCH 1/4] add uploadToWikidata for google and trove --- bull/commons-queue/consumer.js | 31 +++- bull/google-books-queue/consumer.js | 10 +- bull/trove-queue/consumer.js | 24 ++- components/ShowJobInformation.js | 21 ++- server.js | 15 +- utils/helper.js | 240 ++++++++++++++++++++++++++++ 6 files changed, 326 insertions(+), 15 deletions(-) diff --git a/bull/commons-queue/consumer.js b/bull/commons-queue/consumer.js index 79cd8e4..74174ed 100644 --- a/bull/commons-queue/consumer.js +++ b/bull/commons-queue/consumer.js @@ -1,7 +1,11 @@ const config = require("../../utils/bullconfig"); const CommonsQueue = config.getNewQueue("commons-queue"); const winston = require("winston"); -const { downloadFile, uploadToCommons } = require("../../utils/helper"); +const { + downloadFile, + uploadToCommons, + uploadToWikiData, +} = require("../../utils/helper"); const logger = winston.loggers.get("defaultLogger"); CommonsQueue.process(async (job, done) => { @@ -36,9 +40,26 @@ CommonsQueue.process(async (job, done) => { return done(null, true); // return done(new Error(`uploadToCommons: ${commonsResponse}`)); } - process.emit(`commonsJobComplete:${job.id}`, { - status: true, - value: commonsResponse, - }); + const wikiDataResponse = await uploadToWikiData( + job.data.metadata, + commonsResponse.filename + ); + if (wikiDataResponse !== 404) { + process.emit(`commonsJobComplete:${job.id}`, { + status: true, + value: { + commons: commonsResponse, + wikidata: wikiDataResponse, + }, + }); + } else { + process.emit(`commonsJobComplete:${job.id}`, { + status: true, + value: { + commons: commonsResponse, + wikidata: 404, + }, + }); + } return done(null, true); }); diff --git a/bull/google-books-queue/consumer.js b/bull/google-books-queue/consumer.js index 699a540..ac29cfd 100644 --- a/bull/google-books-queue/consumer.js +++ b/bull/google-books-queue/consumer.js @@ -95,6 +95,10 @@ GoogleBooksQueue.process((job, done) => { } done(new Error(errorMessage)); } else { + job.progress({ + step: "Upload To IA", + value: `(${100}%)`, + }); if ( job.data.isUploadCommons !== "true" && job.data.isEmailNotification !== "true" @@ -121,7 +125,11 @@ GoogleBooksQueue.process((job, done) => { step: "Upload to Wikimedia Commons", value: `(${100}%)`, wikiLinks: { - commons: await commonsResponse.value.filename, + commons: await commonsResponse.value.commons.filename, + wikidata: + (await commonsResponse.value.wikidata) !== 404 + ? await commonsResponse.value.wikidata + : 404, }, }); if (job.data.isEmailNotification === "true") { diff --git a/bull/trove-queue/consumer.js b/bull/trove-queue/consumer.js index 4ffaab4..91e68c3 100644 --- a/bull/trove-queue/consumer.js +++ b/bull/trove-queue/consumer.js @@ -103,6 +103,16 @@ TroveQueue.process((job, done) => { } done(new Error(errorMessage)); } else { + job.progress({ + step: "Upload To IA", + value: `(${100}%)`, + }); + if ( + isEmailNotification !== "true" && + job.data.details.isUploadCommons !== "true" + ) { + done(null, true); + } if ( isEmailNotification === "true" && job.data.details.isUploadCommons !== "true" @@ -124,9 +134,14 @@ TroveQueue.process((job, done) => { if (commonsResponse.status === true) { job.progress({ step: "Upload to Wikimedia Commons", - value: `(100%)`, + value: `(${100}%)`, wikiLinks: { - commons: await commonsResponse.value.filename, + commons: await commonsResponse.value.commons + .filename, + wikidata: + (await commonsResponse.value.wikidata) !== 404 + ? await commonsResponse.value.wikidata + : 404, }, }); if (job.data.isEmailNotification === "true") { @@ -134,7 +149,10 @@ TroveQueue.process((job, done) => { EmailProducer( userName, name, - { archiveLink: trueURI, commonsLink: commonsLink }, + { + archiveLink: trueURI, + commonsLink: commonsLink, + }, { archive: true, commons: true } ); } diff --git a/components/ShowJobInformation.js b/components/ShowJobInformation.js index b85aa2a..6a9564a 100644 --- a/components/ShowJobInformation.js +++ b/components/ShowJobInformation.js @@ -144,10 +144,10 @@ const ShowJobInformation = (props) => { - {data.wikimedia_links !== "Not Integrated" ? ( + {data.wikimedia_links.commons !== "Not Integrated" ? ( + + ) : null} + ); diff --git a/server.js b/server.js index d1f4fc4..ba1f194 100644 --- a/server.js +++ b/server.js @@ -169,14 +169,21 @@ app ), uploadStatus: { uploadLink: - job.progress().step.includes("Upload To IA (100%)") && trueURI + (job.progress().step.includes("Upload To IA") || + job.progress().step.includes("Upload to Wikimedia")) && + trueURI ? trueURI : "", isUploaded: jobState === "completed" ? true : false, }, - wikimedia_links: job.progress().wikiLinks?.commons - ? job.progress().wikiLinks.commons - : "Not Integrated", + wikimedia_links: { + commons: job.progress().wikiLinks?.commons + ? job.progress().wikiLinks.commons + : "Not Integrated", + wikidata: job.progress().wikiLinks?.wikidata + ? job.progress().wikiLinks.wikidata + : "Not Integrated", + }, }; res.send( Object.assign( diff --git a/utils/helper.js b/utils/helper.js index 7707b9b..d32496a 100644 --- a/utils/helper.js +++ b/utils/helper.js @@ -315,4 +315,244 @@ module.exports = { return error; } }, + uploadToWikiData: async (metadata, commonsItemFilename) => { + const bot = await Mwn.init({ + apiUrl: "https://www.wikidata.org/w/api.php", + OAuth2AccessToken: metadata.oauthToken, + userAgent: "bub2.toolforge ([[https://bub2.toolforge.org]])", + defaultParams: { + assert: "user", + }, + }); + + async function createEntity(csrf_token) { + try { + const title = metadata.details.volumeInfo.title || ""; + const id = metadata.details.id || ""; + const authorsArr = metadata.details.volumeInfo.authors + ? metadata.details.volumeInfo.authors.join().trim() + : null; + const authors = authorsArr || ""; + // Mapping for the labels/properties defined in `payload` - https://prop-explorer.toolforge.org/ + const payload = { + labels: { + en: { + language: "en", + value: commonsItemFilename, + }, + }, + descriptions: { + en: { + language: "en", + value: title, + }, + }, + claims: { + file_name: [ + { + mainsnak: { + snaktype: "value", + property: "P18", + datavalue: { + value: commonsItemFilename, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + file_url: [ + { + mainsnak: { + snaktype: "value", + property: "P4765", + datavalue: { + value: `https://commons.wikimedia.org/wiki/File:${commonsItemFilename}`, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + commons_category: [ + { + mainsnak: { + snaktype: "value", + property: "P373", + datavalue: { + value: "Bub.wikimedia", + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + inventory_number: id + ? [ + { + mainsnak: { + snaktype: "value", + property: "P217", + datavalue: { + value: id, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + collection: [ + { + mainsnak: { + snaktype: "value", + property: "P195", + datavalue: { + value: { + "entity-type": "item", + "numeric-id": 39162, + id: "Q39162", //wikidataID for 'opensource' + }, + type: "wikibase-entityid", + }, + }, + type: "statement", + rank: "normal", + }, + ], + title: title + ? [ + { + mainsnak: { + snaktype: "value", + property: "P1476", + datavalue: { + value: { + text: title, + language: "en", + }, + type: "monolingualtext", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + name: title + ? [ + { + mainsnak: { + snaktype: "value", + property: "P2561", + datavalue: { + value: { + text: title, + language: "en", + }, + type: "monolingualtext", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + file_format: [ + { + mainsnak: { + snaktype: "value", + property: "P2701", + datavalue: { + value: { + "entity-type": "item", + "numeric-id": 42332, + id: "Q42332", // wikidataID for PDF + }, + type: "wikibase-entityid", + }, + }, + type: "statement", + rank: "normal", + }, + ], + author_name: authors + ? [ + { + mainsnak: { + snaktype: "value", + property: "P2093", + datavalue: { + value: authors, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ] + : undefined, + URL: [ + { + mainsnak: { + snaktype: "value", + property: "P2699", + datavalue: { + value: `https://commons.wikimedia.org/wiki/File:${commonsItemFilename}`, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + copyright_status: [ + { + mainsnak: { + snaktype: "value", + property: "P6216", + datavalue: { + value: { + "entity-type": "item", + "numeric-id": 6938433, + id: "Q6938433", // wikidataID for CC0 license + }, + type: "wikibase-entityid", + }, + }, + type: "statement", + rank: "normal", + }, + ], + }, + }; + + const res = await bot.request({ + action: "wbeditentity", + new: "item", + summary: "bub2.toolforge.org: upload commons item to wikidata", + tags: "wikimedia-commons-app", + data: JSON.stringify(payload), + token: csrf_token, + }); + logger.log({ + level: "info", + message: `uploadToWikidata: Upload of ${commonsItemFilename} metadata to wikidata successful`, + }); + return res.entity.id; + } catch (error) { + logger.log({ + level: "error", + message: `uploadToWikidata:${error}`, + }); + return 404; + } + } + const csrf_token = await bot.getCsrfToken(); + return await createEntity(csrf_token); + }, }; From 9211398ef8fe0d961f8ca21f8b52257a1f36b502 Mon Sep 17 00:00:00 2001 From: Okereke Chinweotito Date: Wed, 7 Feb 2024 10:25:48 +0100 Subject: [PATCH 2/4] change inventory number to internet archive id --- utils/helper.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/helper.js b/utils/helper.js index d32496a..e43cedc 100644 --- a/utils/helper.js +++ b/utils/helper.js @@ -390,12 +390,12 @@ module.exports = { rank: "normal", }, ], - inventory_number: id + internet_archive_id: id ? [ { mainsnak: { snaktype: "value", - property: "P217", + property: "P724", datavalue: { value: id, type: "string", From 8fff4579c8f9fc5933db639f060327cb525145cd Mon Sep 17 00:00:00 2001 From: Okereke Chinweotito Date: Wed, 14 Feb 2024 21:35:49 +0100 Subject: [PATCH 3/4] add uploadToWikisource --- bull/commons-queue/consumer.js | 44 +++++++----- bull/google-books-queue/consumer.js | 4 ++ components/ShowJobInformation.js | 21 +++++- server.js | 13 +++- utils/helper.js | 103 +++++++++++++++++++++++++++- 5 files changed, 160 insertions(+), 25 deletions(-) diff --git a/bull/commons-queue/consumer.js b/bull/commons-queue/consumer.js index 74174ed..bb6d84b 100644 --- a/bull/commons-queue/consumer.js +++ b/bull/commons-queue/consumer.js @@ -5,8 +5,8 @@ const { downloadFile, uploadToCommons, uploadToWikiData, + uploadToWikiSource, } = require("../../utils/helper"); -const logger = winston.loggers.get("defaultLogger"); CommonsQueue.process(async (job, done) => { const downloadFileRes = await downloadFile( @@ -15,49 +15,57 @@ CommonsQueue.process(async (job, done) => { ); if (downloadFileRes.writeFileStatus !== 200) { - logger.log({ - level: "error", - message: `downloadFile: ${downloadFileRes}`, - }); process.emit(`commonsJobComplete:${job.id}`, { status: false, value: null, }); return done(null, true); - // return done(new Error(`downloadFile: ${downloadFileRes}`)); } const commonsResponse = await uploadToCommons(job.data.metadata); if (commonsResponse.fileUploadStatus !== 200) { - logger.log({ - level: "error", - message: `uploadToCommons: ${commonsResponse}`, - }); process.emit(`commonsJobComplete:${job.id}`, { status: false, value: null, }); return done(null, true); - // return done(new Error(`uploadToCommons: ${commonsResponse}`)); } const wikiDataResponse = await uploadToWikiData( job.data.metadata, commonsResponse.filename ); if (wikiDataResponse !== 404) { - process.emit(`commonsJobComplete:${job.id}`, { - status: true, - value: { - commons: commonsResponse, - wikidata: wikiDataResponse, - }, - }); + const wikisourceResponse = await uploadToWikiSource( + job.data.metadata, + wikiDataResponse + ); + + if (wikisourceResponse.fileUploadStatus === 200) { + process.emit(`commonsJobComplete:${job.id}`, { + status: true, + value: { + commons: commonsResponse, + wikidata: wikiDataResponse, + wikisource: wikisourceResponse, + }, + }); + } else { + process.emit(`commonsJobComplete:${job.id}`, { + status: true, + value: { + commons: commonsResponse, + wikidata: wikiDataResponse, + wikisource: 404, + }, + }); + } } else { process.emit(`commonsJobComplete:${job.id}`, { status: true, value: { commons: commonsResponse, wikidata: 404, + wikisource: 404, }, }); } diff --git a/bull/google-books-queue/consumer.js b/bull/google-books-queue/consumer.js index ac29cfd..dd8cf52 100644 --- a/bull/google-books-queue/consumer.js +++ b/bull/google-books-queue/consumer.js @@ -130,6 +130,10 @@ GoogleBooksQueue.process((job, done) => { (await commonsResponse.value.wikidata) !== 404 ? await commonsResponse.value.wikidata : 404, + wikisource: + (await commonsResponse.value.wikisource) !== 404 + ? await commonsResponse.value.wikisource.filename + : 404, }, }); if (job.data.isEmailNotification === "true") { diff --git a/components/ShowJobInformation.js b/components/ShowJobInformation.js index 6a9564a..01d247c 100644 --- a/components/ShowJobInformation.js +++ b/components/ShowJobInformation.js @@ -18,7 +18,8 @@ const ShowJobInformation = (props) => { const styles = { root: { maxWidth: 365, - height: "fit-content", + height: "95vh", + overflow: "scroll", }, cardContentContainer: { height: "200px", @@ -177,6 +178,24 @@ const ShowJobInformation = (props) => { ) : null} + + + {data.wikimedia_links.wikisource !== "Not Integrated" ? ( + + + + ) : null} + ); diff --git a/server.js b/server.js index fae57b4..c95c0b3 100644 --- a/server.js +++ b/server.js @@ -180,9 +180,16 @@ app commons: job.progress().wikiLinks?.commons ? job.progress().wikiLinks.commons : "Not Integrated", - wikidata: job.progress().wikiLinks?.wikidata - ? job.progress().wikiLinks.wikidata - : "Not Integrated", + wikidata: + job.progress().wikiLinks?.wikidata && + job.progress().wikiLinks.wikidata !== 404 + ? job.progress().wikiLinks.wikidata + : "Not Integrated", + wikisource: + job.progress().wikiLinks?.wikisource && + job.progress().wikiLinks.wikisource !== 404 + ? job.progress().wikiLinks.wikisource + : "Not Integrated", }, }; res.send( diff --git a/utils/helper.js b/utils/helper.js index e43cedc..c8c958d 100644 --- a/utils/helper.js +++ b/utils/helper.js @@ -295,9 +295,6 @@ module.exports = { title, metadata.commonsMetadata ); - if (await response.filename) { - await fs.promises.unlink(commonsFilePayload); - } logger.log({ level: "info", message: `uploadToCommons: Upload of ${metadata.IAIdentifier} to commons successful`, @@ -545,6 +542,7 @@ module.exports = { }); return res.entity.id; } catch (error) { + await fs.promises.unlink("commonsFilePayload.pdf"); logger.log({ level: "error", message: `uploadToWikidata:${error}`, @@ -555,4 +553,103 @@ module.exports = { const csrf_token = await bot.getCsrfToken(); return await createEntity(csrf_token); }, + uploadToWikiSource: async (metadata, commonsItemFilename) => { + try { + const bot = await Mwn.init({ + apiUrl: "https://en.wikisource.org/w/api.php", + OAuth2AccessToken: metadata.oauthToken, + userAgent: "bub2.toolforge ([[https://bub2.toolforge.org]])", + defaultParams: { + assert: "user", + }, + }); + + const commonsFilePayload = "commonsFilePayload.pdf"; + const title = metadata.details.volumeInfo.title || metadata.name; + const response = await bot.upload( + commonsFilePayload, + `bub2_${title}`, + "" + ); + console.log("response:", response); + if (await response.filename) { + await fs.promises.unlink(commonsFilePayload); + } + logger.log({ + level: "info", + message: `uploadToWikiSource: Upload of ${commonsItemFilename} to wikisource successful`, + }); + return { + fileUploadStatus: 200, + filename: response.filename, + }; + } catch (error) { + await fs.promises.unlink("commonsFilePayload.pdf"); + logger.log({ + level: "error", + message: `uploadToWikiSource: ${error}`, + }); + return error; + } + }, + + updateWikiData: async (metadata, wikidataId, wikisourceIndex) => { + try { + const bot = await Mwn.init({ + apiUrl: "https://www.wikidata.org/w/api.php", + OAuth2AccessToken: metadata.oauthToken, + userAgent: "bub2.toolforge ([[https://bub2.toolforge.org]])", + defaultParams: { + assert: "user", + }, + }); + + const payload = { + claims: { + wikisource_index_page: [ + { + mainsnak: { + snaktype: "value", + property: "P1957", + datavalue: { + value: wikisourceIndex, + type: "string", + }, + }, + type: "statement", + rank: "normal", + }, + ], + }, + }; + + const csrf_token = await bot.getCsrfToken(); + const res = await bot.request({ + action: "wbeditentity", + id: wikidataId, + data: JSON.stringify(payload), + summary: "Update wikisource index page from bub2.toolforge.org", + token: csrf_token, + }); + + logger.log({ + level: "info", + message: `updateWikiData: Update of Wikidata item ${wikidataId} successful`, + }); + + return { + success: true, + entityId: wikidataId, + }; + } catch (error) { + logger.log({ + level: "error", + message: `updateWikiData: ${error}`, + }); + return { + success: false, + error: error, + }; + } + }, }; From d5040e7a9f14b9fd557ac5da87ca1ea4d0e43877 Mon Sep 17 00:00:00 2001 From: Okereke Chinweotito Date: Wed, 14 Feb 2024 22:20:06 +0100 Subject: [PATCH 4/4] add wikisource to trove --- bull/trove-queue/consumer.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bull/trove-queue/consumer.js b/bull/trove-queue/consumer.js index 91e68c3..ef9cd85 100644 --- a/bull/trove-queue/consumer.js +++ b/bull/trove-queue/consumer.js @@ -142,6 +142,11 @@ TroveQueue.process((job, done) => { (await commonsResponse.value.wikidata) !== 404 ? await commonsResponse.value.wikidata : 404, + wikisource: + (await commonsResponse.value.wikisource) !== 404 + ? await commonsResponse.value.wikisource + .filename + : 404, }, }); if (job.data.isEmailNotification === "true") {