From 4a3fa81d8023ebdfe209b550edcf1257064d8663 Mon Sep 17 00:00:00 2001 From: Thorsten Vitt Date: Mon, 30 Mar 2020 23:59:10 +0200 Subject: [PATCH 1/5] A promise generating transcript function. This currently mirrors renderToPhantom but return a promise instead of calling PhantomJS's callback, making it usable from, e.g., puppeteer. Cf. #13 --- svg_rendering/page/js-gen/transcript-generation.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/svg_rendering/page/js-gen/transcript-generation.js b/svg_rendering/page/js-gen/transcript-generation.js index 9d06882..e940506 100644 --- a/svg_rendering/page/js-gen/transcript-generation.js +++ b/svg_rendering/page/js-gen/transcript-generation.js @@ -208,6 +208,20 @@ var transcriptGeneration = (function () { } }); } + + transcriptGeneration.createPromise = function createPromise(transcript, links) { + return new Promise((resolve, reject) => { + transcriptGeneration.createDiplomaticSvg(transcript, (diploSvg) => { + const result = {svg: serialize(diploSvg), overlay: undefined}; + if (links) { + const overlaySvg = transcriptGeneration.createFacsimileOverlaySvg(diploSvg, links); + result.overlay = serialize(overlaySvg); + overlaySvg.parentNode.removeChild(overlaySvg); + } + resolve(result); + }); + }); + } return transcriptGeneration; })(); From ea299d7b599099889f463e515face2efe176a691 Mon Sep 17 00:00:00 2001 From: Thorsten Vitt Date: Tue, 31 Mar 2020 00:01:31 +0200 Subject: [PATCH 2/5] Render script using puppeteer --- package-lock.json | 315 ++++++++++++++++++++++++++++++++++++++++ rendersvgs-puppeteer.js | 51 +++++++ 2 files changed, 366 insertions(+) create mode 100644 package-lock.json create mode 100644 rendersvgs-puppeteer.js diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..5edcb26 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,315 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "@types/mime-types": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.0.tgz", + "integrity": "sha1-nKUs2jY/aZxpRmwqbM2q2RPqenM=" + }, + "agent-base": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" + }, + "async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, + "buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "extract-zip": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.7.0.tgz", + "integrity": "sha512-xoh5G1W/PB0/27lXgMQyIhP5DSY/LhoCsOyZgb+6iMmRtCwVBo55uKaMoEYrDCKQhWvqEip5ZPKAc6eFNyf/MA==", + "requires": { + "concat-stream": "^1.6.2", + "debug": "^2.6.9", + "mkdirp": "^0.5.4", + "yauzl": "^2.10.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, + "fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", + "requires": { + "pend": "~1.2.0" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "https-proxy-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "requires": { + "agent-base": "5", + "debug": "4" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" + }, + "mime-db": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==" + }, + "mime-types": { + "version": "2.1.26", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", + "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "requires": { + "mime-db": "1.43.0" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "mkdirp": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.4.tgz", + "integrity": "sha512-iG9AK/dJLtJ0XNgTuDbSyNS3zECqDlAhnQW4CsNxBG3LQJBbHmRX1egw39DmtOdCAqY+dKXV+sgPgilNWUKMVw==", + "requires": { + "minimist": "^1.2.5" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, + "pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "puppeteer": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-2.1.1.tgz", + "integrity": "sha512-LWzaDVQkk1EPiuYeTOj+CZRIjda4k2s5w4MK4xoH2+kgWV/SDlkYHmxatDdtYrciHUKSXTsGgPgPP8ILVdBsxg==", + "requires": { + "@types/mime-types": "^2.1.0", + "debug": "^4.1.0", + "extract-zip": "^1.6.6", + "https-proxy-agent": "^4.0.0", + "mime": "^2.0.3", + "mime-types": "^2.1.25", + "progress": "^2.0.1", + "proxy-from-env": "^1.0.0", + "rimraf": "^2.6.1", + "ws": "^6.1.0" + } + }, + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "ws": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", + "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", + "requires": { + "async-limiter": "~1.0.0" + } + }, + "yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", + "requires": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + } + } +} diff --git a/rendersvgs-puppeteer.js b/rendersvgs-puppeteer.js new file mode 100644 index 0000000..12d2267 --- /dev/null +++ b/rendersvgs-puppeteer.js @@ -0,0 +1,51 @@ +const puppeteer = require('puppeteer'); +const fs = require('fs'); +const fsP = fs.promises; + + +const backend = process.argv[2], + input = process.argv[3], + output = process.argv[4], + links = process.argv[5], + linkout = process.argv[6]; + +if (process.argv.length !== 5 && process.argv.length !== 7) { + console.log('Usage: node rendersvgs.js backend input output [links linkoutput]'); +} else { + + (async () => { + try { + const browser = await puppeteer.launch({args: ['--no-sandbox'], /* headless: false, slowMo: 100, devtools: true, */ }); + const page = await browser.newPage(); + + page.on('console', msg => { + for (let i = 0; i < msg.args().length; ++i) + console.log(`${i}: ${msg.args()[i]}`); + }); + + await page.goto(backend, {waitUntil: "domcontentloaded"}); + + const transcript = await fsP.readFile(input, { encoding: "utf-8"}), + imagelinks = links? fsP.readFile(links, {encoding: "utf-8"}) : undefined; + + await page.waitForFunction("function() {return typeof(transcriptGeneration.createPromise) === 'function'}") + await page.$eval("#preload", el => el.parentNode.removeChild(el)); + const result = await page.evaluate((t, i) => { return transcriptGeneration.createPromise(t, i); }, transcript, imagelinks); + + await fsP.writeFile(output, result.svg, {encoding: "utf-8"}); + if (linkout) { + await fsP.writeFile(linkout, result.overlay, {encoding: "utf-8"}); + } + + await page.pdf({path: output.replace(/.svg$/, ".pdf"), + format: 'A4', + margin: '6pt'}); + await page.screenshot({path: output.replace(/.svg$/, ".png"), + fullPage: true}); + browser.close(); + } catch (e) { + console.error(e); + } + + })(); +} From f9b74d8fb86649ea86ab942244efb4982c0572e9 Mon Sep 17 00:00:00 2001 From: Thorsten Vitt Date: Tue, 31 Mar 2020 10:36:27 +0200 Subject: [PATCH 3/5] Integrated puppeteer based rendering into build --- .gitignore | 2 + package-lock.json | 4 +- package.json | 21 ++++++++ pom.xml | 28 +++++++++-- .../gen/DiplomaticConversion.java | 50 +++++++++++-------- 5 files changed, 80 insertions(+), 25 deletions(-) create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 9ec08da..83b083f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ target .ipynb_checkpoints .ropeproject __pycache__ +node +node_modules diff --git a/package-lock.json b/package-lock.json index 5edcb26..619e325 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,8 @@ { - "requires": true, + "name": "faust-gen", + "version": "1.0.0", "lockfileVersion": 1, + "requires": true, "dependencies": { "@types/mime-types": { "version": "2.1.0", diff --git a/package.json b/package.json new file mode 100644 index 0000000..9778d17 --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "name": "faust-gen", + "version": "1.0.0", + "description": "The Faust-Edition web application builds on data that is automatically pre-generated from the original XML files. This project integrates the various generation and upload processes (except for the images, actually).", + "main": "rendersvgs-puppeteer.js", + "dependencies": { + "puppeteer": "^2.1.1" + }, + "devDependencies": {}, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/faustedition/faust-gen.git" + }, + "bugs": { + "url": "https://github.com/faustedition/faust-gen/issues" + }, + "homepage": "https://github.com/faustedition/faust-gen#readme" +} diff --git a/pom.xml b/pom.xml index 467ff04..1fb2ead 100644 --- a/pom.xml +++ b/pom.xml @@ -226,7 +226,7 @@ - + + + com.github.eirslett + frontend-maven-plugin + 1.9.1 + + v12.16.1 + + + + install node and npm + install-node-and-npm + + + npm install + npm + + + - + @@ -320,7 +339,10 @@ - -Dphantomjs.binary=${project.build.directory}/dependency/slimerjs-1.0.0-beta.1/slimerjs + + + -Dnode.binary=${project.basedir}/node/node + -Dnode.script=${project.basedir}/rendersvgs-puppeteer.js -Dfaust.diplo.allowedFailures=${faust.diplo.allowedFailures} -Dfaust.diplo.server=${faust.diplo.server} -Dfaust.diplo.port=${faust.diplo.port} diff --git a/src/main/java/net/faustedition/gen/DiplomaticConversion.java b/src/main/java/net/faustedition/gen/DiplomaticConversion.java index baf65a8..8039f6b 100644 --- a/src/main/java/net/faustedition/gen/DiplomaticConversion.java +++ b/src/main/java/net/faustedition/gen/DiplomaticConversion.java @@ -72,8 +72,6 @@ public class DiplomaticConversion { private static boolean onlyWebServer; - private static ImmutableList baseCmdLine; - public static class TranscriptPage { public final Document document; private final String page; @@ -133,17 +131,11 @@ public boolean buildSVGs() { logger.fine("Converting " + this); final Path resolvedSvgPath = diplomatic_path.resolve(getPagePath("svg")); resolvedSvgPath.getParent().toFile().mkdirs(); - final ArrayList arguments = Lists.newArrayList( - System.getProperty("phantomjs.binary", "/usr/local/bin/phantomjs"), - "rendersvgs.js", - serverURL, - getJsonPath().toString(), - resolvedSvgPath.toString()); - if (debugPhantomJS) - arguments.add(1, "--debug=errors"); - if (arguments.get(0).contains("slimerjs")) - arguments.add(1, "--headless"); - + final ArrayList arguments = getRenderCommandLine(); + + arguments.add(getJsonPath().toString()); + arguments.add(resolvedSvgPath.toString()); + final Optional imageLinkPath = getImageLinkPath(); if (imageLinkPath.isPresent()) { arguments.add(imageLinkPath.get().toString()); @@ -183,7 +175,28 @@ private Path getJsonPath() { } - public static class Document { + private static ArrayList getRenderCommandLine() { + String renderScript = System.getProperty("node.script"); + final String renderBinary; + if (renderScript == null) { + renderBinary = System.getProperty("phantomjs.binary", "/usr/local/bin/phantomjs"); + renderScript = "rendersvgs.js"; + } else { + renderBinary = System.getProperty("node.binary", "node"); + } + + final ArrayList arguments = Lists.newArrayList( + renderBinary, + renderScript, + serverURL); + if (debugPhantomJS) + arguments.add(1, "--debug=errors"); + if (arguments.get(0).contains("slimerjs")) + arguments.add(1, "--headless"); + return arguments; + } + + public static class Document { private final Path path; /** The base faust:// uri for the transcripts of this document */ @@ -231,13 +244,8 @@ public static void main(final String[] args) throws IOException { try { serverURL = new URL("http", "localhost", webServer.getListeningPort(), "/transcript-generation.html").toString(); logger.info(MessageFormat.format("Web server runs on {0}", serverURL)); - baseCmdLine = ImmutableList.of( - System.getProperty("phantomjs.binary", "/usr/local/bin/phantomjs"), - "--profile", profile.toString(), - debugPhantomJS? "--debug=true" : "", - "rendersvgs.js", - serverURL); - logger.info(() -> "PhantomJS command line: " + String.join(" ", baseCmdLine) + " [ ]"); + List baseCmdLine = getRenderCommandLine(); + logger.info(() -> "Render script command line: " + String.join(" ", baseCmdLine) + " [ ]"); if (onlyWebServer) { From 1e56d91a7dbf0b429fe55a7ff7cc4cdac6a6ef38 Mon Sep 17 00:00:00 2001 From: Thorsten Vitt Date: Tue, 31 Mar 2020 16:25:39 +0200 Subject: [PATCH 4/5] timeout for the rendering function --- rendersvgs-puppeteer.js | 82 +++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 35 deletions(-) diff --git a/rendersvgs-puppeteer.js b/rendersvgs-puppeteer.js index 12d2267..6b1090f 100644 --- a/rendersvgs-puppeteer.js +++ b/rendersvgs-puppeteer.js @@ -13,39 +13,51 @@ if (process.argv.length !== 5 && process.argv.length !== 7) { console.log('Usage: node rendersvgs.js backend input output [links linkoutput]'); } else { - (async () => { - try { - const browser = await puppeteer.launch({args: ['--no-sandbox'], /* headless: false, slowMo: 100, devtools: true, */ }); - const page = await browser.newPage(); - - page.on('console', msg => { - for (let i = 0; i < msg.args().length; ++i) - console.log(`${i}: ${msg.args()[i]}`); - }); - - await page.goto(backend, {waitUntil: "domcontentloaded"}); - - const transcript = await fsP.readFile(input, { encoding: "utf-8"}), - imagelinks = links? fsP.readFile(links, {encoding: "utf-8"}) : undefined; - - await page.waitForFunction("function() {return typeof(transcriptGeneration.createPromise) === 'function'}") - await page.$eval("#preload", el => el.parentNode.removeChild(el)); - const result = await page.evaluate((t, i) => { return transcriptGeneration.createPromise(t, i); }, transcript, imagelinks); - - await fsP.writeFile(output, result.svg, {encoding: "utf-8"}); - if (linkout) { - await fsP.writeFile(linkout, result.overlay, {encoding: "utf-8"}); - } - - await page.pdf({path: output.replace(/.svg$/, ".pdf"), - format: 'A4', - margin: '6pt'}); - await page.screenshot({path: output.replace(/.svg$/, ".png"), - fullPage: true}); - browser.close(); - } catch (e) { - console.error(e); - } - - })(); + (async () => { + try { + const browser = await puppeteer.launch({args: ['--no-sandbox'], /* headless: false, slowMo: 100, devtools: true, */ }); + try { + const page = await browser.newPage(); + + page.on('console', msg => { + for (let i = 0; i < msg.args().length; ++i) + console.log(`${i}: ${msg.args()[i]}`); + }); + + await page.goto(backend, {waitUntil: "domcontentloaded"}); + + const transcript = await fsP.readFile(input, {encoding: "utf-8"}), + imagelinks = links ? fsP.readFile(links, {encoding: "utf-8"}) : undefined; + + await page.waitForFunction("function() {return typeof(transcriptGeneration.createPromise) === 'function'}") + await page.$eval("#preload", el => el.parentNode.removeChild(el)); + const result = await page.evaluate((t, i) => { + return Promise.race([ + transcriptGeneration.createPromise(t, i), + new Promise((_, reject) => setTimeout(reject, 30000, 'Rendering took more than 30 seconds')) + ]); + }, transcript, imagelinks); + + await fsP.writeFile(output, result.svg, {encoding: "utf-8"}); + if (linkout) { + await fsP.writeFile(linkout, result.overlay, {encoding: "utf-8"}); + } + + await page.pdf({ + path: output.replace(/.svg$/, ".pdf"), + format: 'A4', + margin: ['2cm', '2cm', '2cm', '2cm'] + }); + await page.screenshot({ + path: output.replace(/.svg$/, ".png"), + fullPage: true + }); + } finally { + browser.close(); + } + } catch (e) { + console.error(e); + } + + })(); } From baabeebd2c1de979766cd80dbbb294b26362654d Mon Sep 17 00:00:00 2001 From: Thorsten Vitt Date: Tue, 31 Mar 2020 18:03:05 +0200 Subject: [PATCH 5/5] write a simple json file with all data to render maybe we can re-use our browser ... --- .../gen/DiplomaticConversion.java | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/faustedition/gen/DiplomaticConversion.java b/src/main/java/net/faustedition/gen/DiplomaticConversion.java index 8039f6b..12dfcde 100644 --- a/src/main/java/net/faustedition/gen/DiplomaticConversion.java +++ b/src/main/java/net/faustedition/gen/DiplomaticConversion.java @@ -48,6 +48,7 @@ import net.sf.saxon.s9api.XdmAtomicValue; import net.sf.saxon.s9api.XsltExecutable; import net.sf.saxon.s9api.XsltTransformer; +import org.codehaus.jackson.map.ObjectMapper; public class DiplomaticConversion { @@ -233,6 +234,63 @@ public Stream transcripts() { } } + public static void writeJob(final Stream documents) throws IOException { + class TranscriptRepr { + private final String json; + private final int pageNo; + private final String links; + private final String out; + + public TranscriptRepr(TranscriptPage page) { + this.json = page.getJsonPath().toString(); + this.pageNo = page.pageNo; + this.out = String.valueOf(page.getPagePath("svg")); + this.links = page.getImageLinkPath().isPresent()? page.getImageLinkPath().get().toString(): null; + + } + + public String getJson() { + return json; + } + + public int getPageNo() { + return pageNo; + } + + public String getLinks() { + return links; + } + + public String getOut() { + return out; + } + } + class DocumentRepr { + + private final String sigil; + + public Object[] getTranscripts() { + return transcripts; + } + + private final Object[] transcripts; + + public DocumentRepr(Document doc) { + this.transcripts = doc.transcripts().map(TranscriptRepr::new).toArray(); + this.sigil = doc.sigil; + } + + public String getSigil() { + return sigil; + } + } + + final Object[] docReprs = documents.map(DocumentRepr::new).toArray(); + final ObjectMapper objectMapper = new ObjectMapper(); + logger.info(docReprs[0].toString()); + objectMapper.writeValue(target.resolve("render-job.json").toFile(), docReprs); + } + public static void main(final String[] args) throws IOException { Properties properties = System.getProperties(); System.setProperty("java.util.logging.SimpleFormatter.format", "%4$s: %5$s%n"); @@ -262,7 +320,9 @@ public static void main(final String[] args) throws IOException { .map(page -> page.writeTranscriptJson()) .collect(Collectors.toList()); ImmutableList allPages = ImmutableList.copyOf(transcriptPages); - + + logger.info("Writing render job description"); + writeJob(getDocuments()); int nThreads = Integer.valueOf(System.getProperty("faust.diplo.threads", "0"));