From a76ab43b87cda948a49fe402a9c8f27d2ce9fc0e Mon Sep 17 00:00:00 2001 From: Romain Gilliotte Date: Wed, 13 May 2020 19:25:18 +0200 Subject: [PATCH] more work on file uploads --- api/src/io.js | 32 +++-- api/src/routers/uploads.js | 114 ++++++++++++---- docker-compose.yml | 3 +- .../input-edition/project-input-edition.html | 34 ++++- .../input-edition/project-input-edition.js | 60 +++++---- .../pages/input-uploads/dropzone.html | 22 ++++ .../pages/input-uploads/dropzone.js | 74 +++++++++++ .../pages/input-uploads/dropzone.scss | 32 +++++ .../input-uploads/project-input-uploads.html | 58 +++++---- .../input-uploads/project-input-uploads.js | 68 +++++----- .../input-uploads/project-input-uploads.scss | 41 ++++-- .../pages/misc-downloads/downloads.html | 47 +++---- .../pages/misc-downloads/downloads.js | 115 ++++++++-------- .../pages/misc-downloads/downloads.scss | 7 +- frontend/src/translation/en/translations.js | 5 +- frontend/src/translation/es/translations.js | 5 +- frontend/src/translation/fr/translations.js | 5 +- frontend/webpack.config.dev.js | 2 +- workers/data/{ => thumbnails}/placeholder.png | Bin workers/data/thumbnails/zipfile.png | Bin 0 -> 10316 bytes workers/package-lock.json | 80 +++++++++++- workers/package.json | 2 + workers/src/helpers/thumbnail.js | 5 +- workers/src/tasks/uploads/archive/process.js | 11 -- workers/src/tasks/uploads/image/process.js | 123 ++++++++---------- workers/src/tasks/uploads/index.js | 25 ++-- workers/src/tasks/uploads/pdf/process.js | 37 ++++-- workers/src/tasks/uploads/zip/process.js | 52 ++++++++ 28 files changed, 736 insertions(+), 323 deletions(-) create mode 100644 frontend/src/components/pages/input-uploads/dropzone.html create mode 100644 frontend/src/components/pages/input-uploads/dropzone.js create mode 100644 frontend/src/components/pages/input-uploads/dropzone.scss rename workers/data/{ => thumbnails}/placeholder.png (100%) create mode 100644 workers/data/thumbnails/zipfile.png delete mode 100644 workers/src/tasks/uploads/archive/process.js create mode 100644 workers/src/tasks/uploads/zip/process.js diff --git a/api/src/io.js b/api/src/io.js index f86d4fa6..f1fb9f80 100644 --- a/api/src/io.js +++ b/api/src/io.js @@ -6,29 +6,37 @@ const Redlock = require('redlock'); class InputOutput { async connect() { - this.mongo = await MongoClient.connect(config.mongo.uri, { useUnifiedTopology: true }); - - this.database = this.mongo.db(config.mongo.database); - this.database - .collection('invitation') - .createIndex({ projectId: 1, email: 1 }, { unique: true }); - this.database.collection('input').createIndex({ sequenceId: 1, 'content.variableId': 1 }); - this.database.collection('user').createIndex({ subs: 1 }); - - // FIXME move this somewhere else + this.mongo = await MongoClient.connect(config.mongo.uri, { + useUnifiedTopology: true, + poolSize: 50, + }); this.redis = new Redis(config.redis.uri); this.redisLock = new Redlock([this.redis]); this.queue = new Bull('workers', config.redis.uri); + + this._createDatabase(); } async disconnect() { this.mongo.close(true); - this.queue.close(); - this.redis.disconnect(); } + + _createDatabase() { + this.database = this.mongo.db(config.mongo.database); + this.database + .collection('invitation') + .createIndex({ projectId: 1, email: 1 }, { unique: true }); + + this.database + .collection('input_upload') + .createIndex({ 'original.sha1': 1 }, { unique: true }); + + this.database.collection('input').createIndex({ sequenceId: 1, 'content.variableId': 1 }); + this.database.collection('user').createIndex({ subs: 1 }); + } } module.exports = { InputOutput }; diff --git a/api/src/routers/uploads.js b/api/src/routers/uploads.js index f2e200aa..cd749659 100644 --- a/api/src/routers/uploads.js +++ b/api/src/routers/uploads.js @@ -5,19 +5,65 @@ const Router = require('@koa/router'); const multer = require('@koa/multer'); const { ObjectId } = require('mongodb'); const JSONStream = require('JSONStream'); +const { Transform, pipeline } = require('stream'); const router = new Router(); router.get('/project/:id/upload', async ctx => { - const forms = await ctx.io.database - .collection('input_upload') - .find( - { projectId: new ObjectId(ctx.params.id) }, - { projection: { 'original.data': 0, 'reprojected.data': 0 } } + const collection = ctx.io.database.collection('input_upload'); + + if (ctx.request.accepts('application/json')) { + const filter = { projectId: new ObjectId(ctx.params.id) }; + const projection = { 'original.data': 0, 'thumbnail.data': 0, 'reprojected.data': 0 }; + const forms = collection.find( + { ...filter, status: { $ne: 'done' } }, + { projection, sort: [['_id', -1]] } ); - ctx.response.type = 'application/json'; - ctx.response.body = forms.pipe(JSONStream.stringify()); + ctx.response.type = 'application/json'; + ctx.response.body = forms.pipe(JSONStream.stringify()); + } else if (ctx.request.accepts('text/event-stream')) { + const options = { batchSize: 1, fullDocument: 'updateLookup' }; + const wpipeline = [ + { $match: { 'fullDocument.projectId': new ObjectId(ctx.params.id) } }, + { + $project: { + 'fullDocument.original.data': 0, + 'fullDocument.thumbnail.data': 0, + 'fullDocument.reprojected.data': 0, + 'updateDescription.updatedFields.thumbnail.data': 0, + 'updateDescription.updatedFields.reprojected.data': 0, + }, + }, + ]; + + const changeLog = collection.watch(wpipeline, options); + const transform = new Transform({ + objectMode: true, + highWaterMark: 1, + transform: (chunk, encoding, callback) => { + if (['insert', 'update'].includes(chunk.operationType)) { + let action = { type: chunk.operationType, id: chunk.documentKey._id }; + + if (action.type === 'insert') { + action.document = chunk.fullDocument; + } else if (action.type === 'update') { + action.update = chunk.updateDescription.updatedFields; + } + + callback(null, `data: ${JSON.stringify(action)}\n\n`); + } + }, + }); + + // Close changelog on all errors (most notably, client disconnect when leaving the page). + pipeline(changeLog, transform, error => void changeLog.close()); + + ctx.response.type = 'text/event-stream'; + ctx.response.body = transform; + } else { + ctx.response.status = 406; + } }); router.get('/project/:projectId/upload/:id', async ctx => { @@ -41,32 +87,35 @@ router.get('/project/:projectId/upload/:id/:name(original|reprojected|thumbnail) if (upload[ctx.params.name]) { ctx.response.type = upload[ctx.params.name].mimeType; ctx.response.body = upload[ctx.params.name].data.buffer; - } else if (ctx.params.name === 'thumbnail') { - ctx.response.type = 'image/png'; - ctx.response.body = await promisify(readFile)('data/placeholder.png'); } }); router.post('/project/:projectId/upload', multer().single('file'), async ctx => { const file = ctx.request.file; - const insertion = await ctx.io.database.collection('input_upload').insertOne({ - status: 'pending_processing', - projectId: new ObjectId(ctx.params.projectId), - original: { - sha1: new Hash('sha1').update(file.buffer).digest(), - name: file.originalname, - size: file.size, - mimeType: file.mimetype, - data: file.buffer, - }, - }); + try { + const insertion = await ctx.io.database.collection('input_upload').insertOne({ + status: 'pending_processing', + projectId: new ObjectId(ctx.params.projectId), + original: { + sha1: new Hash('sha1').update(file.buffer).digest(), + name: file.originalname, + size: file.size, + mimeType: file.mimetype, + data: file.buffer, + }, + }); - await ctx.io.queue.add( - 'process-upload', - { uploadId: insertion.insertedId }, - { attempts: 1, removeOnComplete: true } - ); + await ctx.io.queue.add( + 'process-upload', + { uploadId: insertion.insertedId }, + { attempts: 1, removeOnComplete: true } + ); + } catch (e) { + if (!e.message.includes('duplicate key error')) { + throw e; + } + } ctx.response.status = 204; }); @@ -77,8 +126,19 @@ router.post('/project/:projectId/upload', multer().single('file'), async ctx => // _id: new ObjectId(ctx.params.id), // projectId: new ObjectId(ctx.params.projectId), // }, -// { $set: { inputId: ctx.body.inputId } } +// { $set: { status: 'done' } } // ); + +// ctx.response.status = 204; // }); +router.delete('/project/:projectId/upload/:id', async ctx => { + await ctx.io.database.collection('input_upload').deleteOne({ + _id: new ObjectId(ctx.params.id), + projectId: new ObjectId(ctx.params.projectId), + }); + + ctx.response.status = 204; +}); + module.exports = router; diff --git a/docker-compose.yml b/docker-compose.yml index 66d5645f..6bbd8063 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,12 +10,13 @@ services: MONGO_INITDB_ROOT_PASSWORD: admin volumes: - mongo_data:/data/db + command: --replSet rs0 redis: image: redis ports: - "6379:6379" - + unoconv: image: zrrrzzt/docker-unoconv-webservice ports: diff --git a/frontend/src/components/pages/input-edition/project-input-edition.html b/frontend/src/components/pages/input-edition/project-input-edition.html index 52b23e3c..52d76aa6 100644 --- a/frontend/src/components/pages/input-edition/project-input-edition.html +++ b/frontend/src/components/pages/input-edition/project-input-edition.html @@ -4,10 +4,33 @@
+ +
+
+ + + + +
+
+
+ +
@@ -21,14 +44,15 @@
-
+
+ +

+

+ Formats supportés: Photos, scans ou fax (pdf, jpg, png, tiff), excel (xlsx) ou archives + (zip). +

+
diff --git a/frontend/src/components/pages/input-uploads/dropzone.js b/frontend/src/components/pages/input-uploads/dropzone.js new file mode 100644 index 00000000..4306582e --- /dev/null +++ b/frontend/src/components/pages/input-uploads/dropzone.js @@ -0,0 +1,74 @@ +import angular from 'angular'; +import axios from 'axios'; +require(__scssPath); + +const module = angular.module(__moduleName, []); + +/** + * The highlight counters works around dragenter/leave behaviour w/ child elements + * + * @see https://stackoverflow.com/questions/7110353/html5-dragleave-fired-when-hovering-a-child-element + */ +module.component(__componentName, { + bindings: { + project: '<', + }, + template: require(__templatePath), + controller: class { + constructor() { + this.types = [ + 'application/pdf', + 'image/jpeg', + 'image/png', + 'image/tiff', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/zip', + ]; + + this.inputTypes = this.types.join(', '); + } + + onDragOver(e) { + e.preventDefault(); + e.stopPropagation(); + + this.highlight = e.target; + } + + onDragLeave(e) { + if (e.target === this.highlight) { + e.preventDefault(); + e.stopPropagation(); + + this.highlight = null; + } + } + + onDrop(e) { + e.preventDefault(); + e.stopPropagation(); + + this.highlight = null; + this._handleFiles(e.dataTransfer.files); + } + + onInputChange(e) { + this._handleFiles(e.target.files); + } + + _handleFiles(files) { + const url = `/project/${this.project._id}/upload`; + const options = { headers: { 'Content-Type': 'multipart/form-data' } }; + + for (var i = 0; i < files.length; ++i) { + if (this.types.includes(files[i].type)) { + const formData = new FormData(); + formData.append('file', files[i]); + axios.post(url, formData, options); + } + } + } + }, +}); + +export default module.name; diff --git a/frontend/src/components/pages/input-uploads/dropzone.scss b/frontend/src/components/pages/input-uploads/dropzone.scss new file mode 100644 index 00000000..8c0f1db2 --- /dev/null +++ b/frontend/src/components/pages/input-uploads/dropzone.scss @@ -0,0 +1,32 @@ +dropzone { + & .highlight { + background-color: yellow; + } + + > div { + border: 1px dashed #666; + border-radius: 5px; + padding: 20px; + text-align: center; + font: 16pt bold; + color: #666; + margin-bottom: 20px; + + * { + pointer-events: none; + } + + label { + pointer-events: auto; + color: #66f; + font-weight: normal; + text-decoration: underline; + cursor: pointer; + } + + input[type='file'] { + position: fixed; + top: -100px; + } + } +} diff --git a/frontend/src/components/pages/input-uploads/project-input-uploads.html b/frontend/src/components/pages/input-uploads/project-input-uploads.html index d2146b6e..c314344e 100644 --- a/frontend/src/components/pages/input-uploads/project-input-uploads.html +++ b/frontend/src/components/pages/input-uploads/project-input-uploads.html @@ -1,37 +1,51 @@ -
Drop Files Here
+
- + -
- Nom: - {{upload.original.name}}
+ +
+ +
- Date: - {{upload.createdAt}} +
+ {{upload.original.name}} -
- En attente de traitement -
+ + + Traitement en cours + -
- This file was entered on X -
+
+ + + Saisir + + + + -
- Le traitement à échoué: {{upload.reason}} +
- - Saisir
diff --git a/frontend/src/components/pages/input-uploads/project-input-uploads.js b/frontend/src/components/pages/input-uploads/project-input-uploads.js index 6b5b5ac2..3894c1c7 100644 --- a/frontend/src/components/pages/input-uploads/project-input-uploads.js +++ b/frontend/src/components/pages/input-uploads/project-input-uploads.js @@ -1,9 +1,10 @@ import angular from 'angular'; import uiRouter from '@uirouter/angularjs'; import axios from 'axios'; +import dropzone from './dropzone'; require(__scssPath); -const module = angular.module(__moduleName, [uiRouter]); +const module = angular.module(__moduleName, [uiRouter, dropzone]); module.config($stateProvider => { $stateProvider.state('project.usage.uploads', { @@ -11,7 +12,11 @@ module.config($stateProvider => { component: __componentName, resolve: { uploads: $stateParams => - axios.get(`/project/${$stateParams.projectId}/upload`).then(r => r.data), + axios + .get(`/project/${$stateParams.projectId}/upload`, { + headers: { accept: 'application/json' }, + }) + .then(r => r.data), }, }); }); @@ -23,42 +28,41 @@ module.component(__componentName, { }, template: require(__templatePath), controller: class { - constructor($state, $timeout) { - // fixme: reload controller every 5 seconds. - // Should be an event source, but convenient for testing. - this.stopInterval = $timeout(() => { - $state.reload('project.usage.uploads'); - }, 3000); + constructor($scope) { + this.$scope = $scope; } - }, -}); -module.directive('dropzone', function () { - return { - restrict: 'A', - scope: false, - link: function (scope, element) { - element.bind('dragover', e => { - e.preventDefault(); - e.stopPropagation(); - }); + $onChanges() { + this.eventSource = new EventSource(`/api/project/${this.project._id}/upload`); + this.eventSource.onmessage = this.onMessage.bind(this); + } - element.bind('drop', e => { - e.stopPropagation(); - e.preventDefault(); + $onDestroy() { + this.eventSource.close(); + } - var files = e.dataTransfer.files; - for (var i = 0; i < files.length; ++i) { - const formData = new FormData(); - formData.append('file', files[i]); + onMessage(message) { + const action = JSON.parse(message.data); - axios.post(`/project/${scope.$ctrl.project._id}/upload`, formData, { - headers: { 'Content-Type': 'multipart/form-data' }, - }); + if (action.type === 'insert') { + this.uploads.unshift(action.document); + } else if (action.type === 'update') { + // Update document. + const index = this.uploads.findIndex(u => u._id == action.id); + const upload = this.uploads[index]; + for (let key in action.update) { + upload[key] = action.update[key]; } - }); - }, - }; + + // Remove if done + if (upload.status === 'done') { + this.uploads.splice(index, 1); + } + } + + this.$scope.$apply(); + } + }, }); export default module.name; diff --git a/frontend/src/components/pages/input-uploads/project-input-uploads.scss b/frontend/src/components/pages/input-uploads/project-input-uploads.scss index b1ddf5c8..18221bcb 100644 --- a/frontend/src/components/pages/input-uploads/project-input-uploads.scss +++ b/frontend/src/components/pages/input-uploads/project-input-uploads.scss @@ -1,25 +1,40 @@ project-input-uploads { - div[dropzone] { - border: 2px dashed #bbb; - border-radius: 5px; - padding: 25px; - text-align: center; - font: 20pt bold; - color: #bbb; - margin-bottom: 20px; - } - .card-container { display: grid; grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); gap: 10px; .thumbnail { + position: relative; margin-bottom: 0; - } + padding-bottom: 20px; + + img { + width: 100%; + } + + .loading-img { + width: 100%; + padding-top: 66.66666%; + position: relative; + + i { + position: absolute; + left: 50%; + top: 50%; + margin-left: -5px; + margin-top: -5px; + } + } - img { - width: 100%; + .caption { + > .btn-group, + > .btn { + position: absolute; + right: 5px; + bottom: 5px; + } + } } } } diff --git a/frontend/src/components/pages/misc-downloads/downloads.html b/frontend/src/components/pages/misc-downloads/downloads.html index a98ed8fa..b5e97238 100644 --- a/frontend/src/components/pages/misc-downloads/downloads.html +++ b/frontend/src/components/pages/misc-downloads/downloads.html @@ -1,26 +1,29 @@ -
-
- -
-

{{file.category|translate}}

-

{{file.name}}

-
- - - - -
- - +
+ + +
+
+ +
+

{{file.name}}

+
+ + + - +
+ + + + +
diff --git a/frontend/src/components/pages/misc-downloads/downloads.js b/frontend/src/components/pages/misc-downloads/downloads.js index ccaf9285..adcf7269 100644 --- a/frontend/src/components/pages/misc-downloads/downloads.js +++ b/frontend/src/components/pages/misc-downloads/downloads.js @@ -31,66 +31,75 @@ module.component(__componentName, { const language = this.$rootScope.language; const serviceUrl = this.$rootScope.serviceUrl; - this.files = [ - ...this.project.logicalFrames.map(lf => { - const url = `${serviceUrl}/project/${projectId}/logical-frame/${lf.id}`; + this.sections = [ + { + id: 'logframes', + name: 'shared.logical_frames', + files: this.project.logicalFrames.map(lf => { + const url = `${serviceUrl}/project/${projectId}/logical-frame/${lf.id}`; - return { - id: lf.id, - category: 'project.logical_frame', - name: lf.name, - thumbnail: `${url}.pdf.png?language=${language}`, - main: { - icon: 'fa-file-pdf-o', - key: 'project.download_portrait', - url: `${url}.pdf?language=${language}&orientation=portrait`, - }, - dropdown: [ - { + return { + id: lf.id, + name: lf.name, + thumbnail: `${url}.pdf.png?language=${language}`, + main: { icon: 'fa-file-pdf-o', - key: 'project.download_landscape', - url: `${url}.pdf?language=${language}&orientation=landscape`, + key: 'project.download_portrait', + url: `${url}.pdf?language=${language}&orientation=portrait`, }, - ], - }; - }), - ...this.project.forms.map(ds => { - const url = `${serviceUrl}/project/${projectId}/data-source/${ds.id}`; + dropdown: [ + { + icon: 'fa-file-pdf-o', + key: 'project.download_landscape', + url: `${url}.pdf?language=${language}&orientation=landscape`, + }, + ], + }; + }), + }, + { + id: 'paper', + name: 'project.collection_form_paper', + files: this.project.forms.map(ds => { + const url = `${serviceUrl}/project/${projectId}/data-source/${ds.id}`; - return { - id: ds.id, - category: 'project.collection_form2', - name: ds.name, - thumbnail: `${url}.pdf.png?language=${language}`, - main: { - key: 'project.download_portrait', - icon: 'fa-file-pdf-o', - url: `${url}.pdf?language=${language}&orientation=portrait`, - }, - dropdown: [ - { + return { + id: ds.id, + name: ds.name, + thumbnail: `${url}.pdf.png?language=${language}`, + main: { + key: 'project.download_portrait', icon: 'fa-file-pdf-o', - key: 'project.download_landscape', - url: `${url}.pdf?language=${language}&orientation=landscape`, + url: `${url}.pdf?language=${language}&orientation=portrait`, }, - ], - }; - }), - ...this.project.forms.map(ds => { - const url = `${serviceUrl}/project/${projectId}/data-source/${ds.id}`; + dropdown: [ + { + icon: 'fa-file-pdf-o', + key: 'project.download_landscape', + url: `${url}.pdf?language=${language}&orientation=landscape`, + }, + ], + }; + }), + }, + { + id: 'excel', + name: 'project.collection_form_excel', + files: this.project.forms.map(ds => { + const url = `${serviceUrl}/project/${projectId}/data-source/${ds.id}`; - return { - id: ds.id + 'xls', - category: 'project.collection_form2', - name: ds.name, - thumbnail: `${url}.xlsx.png?language=${language}`, - main: { - key: 'project.download_excel', - icon: 'fa-file-excel-o', - url: `${url}.xlsx?language=${language}`, - }, - }; - }), + return { + id: ds.id + 'xls', + name: ds.name, + thumbnail: `${url}.xlsx.png?language=${language}`, + main: { + key: 'project.download_excel', + icon: 'fa-file-excel-o', + url: `${url}.xlsx?language=${language}`, + }, + }; + }), + }, ]; } }, diff --git a/frontend/src/components/pages/misc-downloads/downloads.scss b/frontend/src/components/pages/misc-downloads/downloads.scss index 7d4cdee5..be4f3352 100644 --- a/frontend/src/components/pages/misc-downloads/downloads.scss +++ b/frontend/src/components/pages/misc-downloads/downloads.scss @@ -1,8 +1,13 @@ downloads { + legend { + font-size: 24px; + } + .grid-container { display: grid; - grid-template-columns: repeat(auto-fill, minmax(250px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); gap: 10px; + margin-bottom: 30px; .thumbnail { margin-bottom: 0; diff --git a/frontend/src/translation/en/translations.js b/frontend/src/translation/en/translations.js index f5788e5a..fd44abb2 100644 --- a/frontend/src/translation/en/translations.js +++ b/frontend/src/translation/en/translations.js @@ -121,6 +121,8 @@ export default { update_invitation: 'Update invitation', }, project: { + original_file: 'Original file', + uploads: 'Paper & Excel forms', no_logframe_yet: 'This project does not have logical frameworks yet', show_all_projects: 'Mostrar todos los proyectos', no_projects: "You don't have any project", @@ -497,7 +499,8 @@ export default { collection_site: 'Collection site', collection_form: 'Data source', - collection_form2: 'Data entry sheet', + collection_form_paper: 'Paper forms', + collection_form_excel: 'Excel forms', collection_form_planning: 'Calendar', collection_form_structure: 'Structure', diff --git a/frontend/src/translation/es/translations.js b/frontend/src/translation/es/translations.js index 10fe04d4..c1348de9 100644 --- a/frontend/src/translation/es/translations.js +++ b/frontend/src/translation/es/translations.js @@ -121,6 +121,8 @@ export default { update_invitation: 'Actualizar invitación', }, project: { + original_file: 'Fichero original', + uploads: 'Formularios en papel & Excel', no_logframe_yet: 'Aún no ha creado ningún marco logico en este proyecto', show_all_projects: 'Display all projects ({{count}})', no_projects: 'No tiene ningún proyecto', @@ -498,7 +500,8 @@ export default { collection_site: 'Lugar de colecta', collection_form: 'Fuente de datos', - collection_form2: 'Formulario', + collection_form_paper: 'Formulario en papel', + collection_form_excel: 'Formulario en Excel', collection_form_planning: 'Calendario', collection_form_structure: 'Estructura', diff --git a/frontend/src/translation/fr/translations.js b/frontend/src/translation/fr/translations.js index ac9b3d4e..19653f50 100644 --- a/frontend/src/translation/fr/translations.js +++ b/frontend/src/translation/fr/translations.js @@ -122,6 +122,8 @@ export default { update_invitation: "Mettre à jour l'invitation", }, project: { + original_file: 'Fichier original', + uploads: 'Formulaires papier & Excel', no_logframe_yet: "Vous n'avez pas encore créé de cadre logique sur ce projet", show_all_projects: 'Afficher tous les projets ({{count}})', no_projects: "Vous n'avez aucun projet", @@ -492,7 +494,8 @@ export default { collection_site: 'Lieu de collecte', collection_form: 'Source de données', - collection_form2: 'Fiche de saisie', + collection_form_paper: 'Fiches de saisie papier', + collection_form_excel: 'Fiches de saisie Excel', collection_form_planning: 'Calendrier', collection_form_structure: 'Structure', diff --git a/frontend/webpack.config.dev.js b/frontend/webpack.config.dev.js index 283b4918..6a681763 100644 --- a/frontend/webpack.config.dev.js +++ b/frontend/webpack.config.dev.js @@ -11,7 +11,7 @@ module.exports = { devServer: { contentBase: path.resolve('public'), port: 8080, - compress: true, + compress: false, disableHostCheck: true, host: '0.0.0.0', proxy: { diff --git a/workers/data/placeholder.png b/workers/data/thumbnails/placeholder.png similarity index 100% rename from workers/data/placeholder.png rename to workers/data/thumbnails/placeholder.png diff --git a/workers/data/thumbnails/zipfile.png b/workers/data/thumbnails/zipfile.png new file mode 100644 index 0000000000000000000000000000000000000000..6f3e81252a8c8510eeaa017bd224b5c23046970b GIT binary patch literal 10316 zcmX|nby$<_`}gQ(w1j|&bfX~Mtsu2I(jg@=K@e$VpdbiJ4W*?-#(+_yBqpHJIl@sA zjt1$J-*~?7@&2(ot{wMwo!1$kxZ}*t3>j#-X#oHLgRzmGB>+GoNqqm4nv(czIX4X= zzL5A^8tRZtbxXb^exUI+vhfE1=vc2FB!J9pP5?mUud$xCbx`)E16_inVZh-up$*8R z8o}hw>$@?3TsP1`qXhI8X%vN;M@9vju9DgENxiYA)|LPbCEoKW)hjcRDj(zc5>y*! zm1bFN4U4>a-S)}o9RcauBrSd>X5UX(64f}r!>){&+F` z`0K$OrVnGmt>3Wr1VNSbwPLLErz84f>d@Bd2C{S}ZY%)q^v#$+mCB{Rz+Z0=$FGq}*8O10BgGQjgt+hTbwRR{q(c#ci5E4K` z?M(0&=`1eJ^+r{HU?wT6+G}L}rqgIagK-M!;fh3Jvwn>~Z&Arpt5jQ-^PsVOy+Js%}i*2PV>^2EG z!M|~~oXrw(2VjloLGD)vxXK`?qZmj?mp5EDjPu#Uwt^sO`pim-cLMCBX9OIb2%hDV; z6Y$W`LS~jWaV5vxOiP(=`NvnS!u1au?p5YPRv3oSk|v)Mcw0SPo+_H#VwDAD8D5xL zE|FKYM*2jF|FywBXE7yga^CnCcsvgxvC^m{ zLtTe5r;ovr(UVQ&Wh3NRDLcj32&QxCJK6qY+|RoxAK3Z?Z9AJ+l;H^qk%+7C_i6J+ zgEm!nWJu9bPV+Llg+aeQS*SBEJ1KsAwS8^q;NCx5yVLt6qpX-sC*4n>p=yHxc zUGUjg(u|3_N>6Z*O3;ZHBg&gK+5krhK|Hn2QtZF4+tKi7_z7R^uH*56z}b3rISkF> zz5SepXYB+T)zNAAO?G&Rd4uLvayYR#nF?T>O={3@SFe6|Dch52{ekP7dVd^^x9ZBJB zfPvTV%e)a6N0CBNl_3F$@x%TcrldRQ3);kZJvW3yH1v{bBzlf2QOLl%FlbGOhhU?y zuqPXz8yh*y^>lE*XLV2erTy*uVMT2rc5Ry~d1sSz%daAD(H7XJs@o*f=f8n5E#+y) z@NDaP*NBIDU@<)!?-!0e$rA=#FlX>Kl8Ap%Zf0+-_$vC zAXH8m9j-?}%-^4XYPoqFiN7KR2Nh1y zQ9{m(`k1N84CKeFmE}a9v<^*lpMJBXOA131t(4ZY4@BiGMT3x*xnZ` z@kY}GimsgeZQ;fv&HmwyFt8=T5_3Xxes(JT-T+*M1NP|a`>KEooW1&kb)>wOc z)b%&a>37h>>?z#xk&41qHNSt-*Z>(^&788zp*R(>J0*s&=%!iH<=CIuWB#Cn%&74X zh#wPxCtc6N^eNCIxH4j4A@n-bJ(S44gg+GmLUe}sE2a(=(LfW z-{NLlKIogzSw0p$Uc%O`k5}mAwm>R(nzKH(kER@3t34=Ni+X=m*?RpQYv*IHuWxjDT7D@V%x|4bbTK@a(678(#Sr1+MQIx^uyFU2J>I27ML=4C;;BgI zJ$7K(8(09D2eLkw&2z(lE)TA9$(+KQ6IDIasrd7g@bQkh|3Zi0>TJiadWf=sbiiRX zOSoW1`&Sq(-?+VL+)q59H4n{ zK!jbQiGde1oDUq-*?Aa`vA9n8!k}y?dY%@LYREn{EVk3j^RUt;vEO5PRXprq=P5ZV z-GYxCZ*ny?OG`g&hBe$I_B~)P9AQ8AW)R31rlEOpkQ1M8uUK**B-%#vI@_mFGE(cbmRBDo(WuWq%6w&ka!#k{8%vqraAwrouY z8v`_K8C>boY!sCMJ4o61Lt?VmZJ#}Ko!Av34QDevo1X(tY+hYsmmXS$?FC3II)SRn z;lE85L*9DZ-T32w6@qRS;M+T5)=u~WM|wvFV6Re5!T7u(H7W*B=A0KAx9BvCaEXi6 zSFVxgtC02Q1Lr&4D5lhW46UO_Qy))nN)$;#d`2F-fm{bXe=53t?-2LqK~QQ&LF=d? zdSP2eg&3OgRGQNYtcfSj=pNdnoYL~ijvy^KyuDz>UxecgRyK*0pSrG}xIYszj|ql| zSJ<^55lq*MiL0GLR0a+*RUd?9LX?1BttRsZ&%T!29d2ghzIJ)DuObZYz8xPLe!=(8 zCnI-`=V&S9-~Ec%z`l4D$8Vz9%crvOMW>?gPB9ctqIFVr;lQtko;;Jqby6|zKW7=X zvw0__>l5(_Q;KNy*&07w51r`gG>wzoYWEjLvOSC(1F)RCxsBy&jgyjA29&a!pj}*UEcmAX02e|p`Sce{LqTrFGe3h}wK8sRe~mz> zYaZtvq?u8^aL8xEQo@0unZ%@Gx<;UmJ*^9M&I3%3_o%O*s5_Vl#Kv=LwfhNXZ|-R* zF@;aGpfDCr+1E9xpJ!ipoqq>_SbUaF+P{K@w|phs0o-P09_WQ5-HwloEf`3{yKBcs zNx5^kho}r)j@&|?*juXmJTHyl3qJZ92a~@@fjNz0oK6K-FPdpir~6$k5Md8uSB#~c z%e+%5Wca1E3O`DC*gcZ*V*07)<7HQgWF<-c zB z0FVIVuX=Okw2r#?B6u(5QfGv5=ze~|h`UNoYhMk%x?Bu@aU+OMKaV)f6e2+cJ6pBv zRR1DvJZgHxatKqk+sL{wVmS}KNxepTS6fN>>yfFlvSO+ZOntCNCC~k^yFyjqM&o?J zbZ=(Fj6uj4H5;!;>~1jFxqjxeL>^&w8}PQ^7WbAh!xajj$R$~de%RGrT4iCO7LdP= z8g$E3<>PQQ9#fO_QYq z{O{i?G$B*Vi~oS{+?!4u6}JVv`i$*4nT}J}QPUmz6(jXbH^XaEk!9+hK0ce6xq=kj z>{CK7rdaOEtg_UXLyM~uNvXV-!nR&DjIlM84?cQ>@la9*0pn`ou@$4N)fc8|VYwIY zHkyc8t7yUwe7l2&>puuGrjN_wj6@8a2=861^UJ87SD0U7Bef)^L1#1jDTR(kpqeBy zZmlXS7yrK`dH3HjcQT=V zGRxEihEj`ARSa_dis5Am90Nub?p#vjGQZI;AvD1wARk8O;?>T;tlLgq|%jDzs9oKcJ5!RzEkH1olN>E@E1- zMmy8p8p~o%S zOxLw%)hE(mzMH<6GYxz-n+S78_*PHA`b7CNXr2AX+*nX)w4V;dlYSbT&M`;0alm8# zkGbcr`k*QWJOGSAM(?M%?DOIeZ>{h}@8<0gc}c`$eI~ue5RnrnJ1oW^65P?@2US2? z-tuJ**Hn7i;G{Vx`IW!tN4}bV5f)Dm{`gNov`vKwpE4EOO_uj)p`a;63ZU049QVdV zT1h-vN2KKQKrA%)ZrXM?ceZQf&`C3cw<_Io-nncL$-=`sq3|>5ccFiC8z%5%x-&C7 zUmZ?JCtf_`z2rTfaVX%6sb~&mZBe(7oQSpPJoxbK(p{fd#^QRvpSs#*Mtsod0EIjQ z@Sc?nk;kN|giJYB)>Du!S1^;2+j*-}ESF$;9S3Aw>uYOaXHmCi6BaEfA+wRO3hz6c zp0@U1_3&SK>zmYO_s#67;k!3Dve|gfso@~-@$|fu*F-L1wO!o8+oQDikn~8J`jCB1 zO~1WI{ZUuPxAvZnm~k^|qsQMnETeGy`(u>U?-m(RMw5qkTsSIaGp@1lgcx2;OnNL* z9C%ORWIG6AEyMuuE*O8}x)Q^4=->3sPyyZ>`fpJ8L|GIcHn;va?u1z6DmSE5U*=~X zd0-W73U|Rlv$kKFz4Fa2nR<*FZiM{+M)k5|UL*6Zd!aZgl=wkUUl0TV7SCbgutG1G z#e524?ky)bAw7DT|CN*5KD)V190C$>v^Aww5863#V*3$xd_j_HDhBxNFOfdz&VVbU zhVa5cl#t?e#A+~v5oG`(&Kixq3(ly8xAE=gYR=x#zM9K@$b0yB`N`mzeFMdh-ai1p zm&Nhyx}`diR_rUse{X}zZ!(=gF|$KVT%Hs@g}Rd&>kwaLgP*(}RzF;AG! zgR^(rOM&oLir9l}dASz9T*X_{?;oiTv7~xDMj1RG zGD}YiCVV;%+xjPsMXox{53z)j5Y-*a_j2RWDwn`p_zUOPo3{pP#69dZMrFQr5IMVn1TN=30h^T$YcQLMUA2rKWuM#nDIDD^2_Ba%lW8M+%M zB0J|lemjhO3&^b;k(xbApDq-cCtR}b&%t-Izrvh2Tg~^j__cyb|6P!Y&Y8uEukwBKZvEtbb4ymg9hrLLSX=bLAG)v2 zk2=@QbHR5q?d0)_-jGZBxPP_$$M`)P{+~=xY|Y^Etd!7GliACX*6r5;u3(#%xX9w} zpqd0wFdx_y>QeBiJ*?tQTo}Pa&r3GJ@Cw=!0n}?n@y|90lkLg?E}q=}_m3hNWt`Za z=6%ebsq?9Ay`6w-jnau7&9s75IP%aYq${(dJ5d$cGe z8My0Fkxf2BLIhB45#{e<5hPIwD@;r)ZcI6 z{x$C5hFDii42HxUtX+*jB40uW-#81^{eg}Y zk`$94!8Q||Q3wo{a6lnHD?M0)-0&~jyQA#*z1HzRF~Cht(qo=Gmobv!#d3R2@c3ts zpi2T%`*Bu|s?4ewX^k`=m>Iob{WfcT0L8I+yEn)Up0KZgXA`& z5>L+D??%>c-ZqQhh4r3 z%3-BL>*HfVIK}@ua|=re*%*7n&bxY9xOQ&q8J;Nj7Ivfb@btlue*!c&!q_7>0=~90 z?K5a|p5I)Vqn_Wl`lL#}>^GF7DJSQoV(dVR3Tp9`pZ$zQ*3rDk*yJpS|9HZV&nAv- zc&`aDaB{w9+mz4-uGxc+OEV8*EsjUlE9V~FA(Q>>6wJ*BmdK;A!bxbJ84trJGi|u0 zz9*?Ca|z(!IO=%haCe9WGx|=M<>x(ClWD}G-8cjjE`~leuAZTeZB88_>_9IXrls1L&@V1B;_ctO zt-L)vLVbQm^$~=jh<}^<8hgb=8S{#Pr8kI7dgj+!R6Q;svnDMg!UeU>HU^s0N`ur? ziR89>y;KWS$Hv@a9-NL_Kk>qa3R>d<@3jj(mv1|L<~5pCv~e3Dfk=^|3jVup5Tq$2 zz%D@aE%L5~E8zZM2hyh+k#}1G@{Bb)mw)!?#qmkk6SX60ya(j^KW`ic-ct~KHg4pZ`b>;8fpMM4GmI>4Y&bmkq;Ba z2}Ay_8}>$lEcvfgJ(y*YMU|TwRnk$l7!)`8WNT1;=1nX`q_5HL#(6L#F> z^G=O@o@`xAC>3R1=<96<%E1e18SnP%{NfY`dHw3)It7F?R+#}jhrw^t6v8ZG&8r)g z6e5>Ncbx#l%e(y5@|XQYCXrRV;;T;t3@_w&OUS5C$3-TEFaf(x_|NUgFEIJ`T}m}n z@X`$NsECash4MWT3p}X4-;yx!%Kh$aZ@RZNh^8D=N+*oTikN)CxT?kHJ*3D|@-aTL z__x#Q8m?o*Pe+R=-`&IY>c~zf^znoTye5?2->#Gh_3*xTuLFv+JtRH&M`uK}TPcMf zZyL-{2KII;UXA4Vb_>c1qDCi}IxUiKKWjO!s7}A}yxEi)KRGX0Q_Z;K!;FVfNZ1f; zUd4iX6h&SMU2A~s1dWz~`)^42oqPSPubt`r0}}**%iY2gsyx0NRSNR{JQKpCstooF zDYD1e{P-V1EBf#{@>s^jb&W3c$ni7Jz;n4Y&yXkw+&oY3>11TfOX98^@j5Lp8ow-f z1pNo~<0+-dJNX49tipKu2aUGsouXQXPio^aM3cao)qDzoH@K zm}CPgcUZSkozqZwY^)C-m~a zyL)w$NbJXaWxOZ`O`Pq1ZwOI`eR+B*dTw|aT!X^S7FCCHI9rY;6y3UU2jQ%AC&?@NIFeEJBd68;|e4e8FZI zs>e$NVh0CKYy*sKS$K{E*y3O^p5q9XI@S8b4p_yoMHDyVnD8IG-^$~URS$K8)zh56 zvJL^hJscTKL;M;Uk%vf;f~UMQGxpH;|KNveerG?IM2^)P9g@!b^O64$S*JbyOtd1) zu!pw+&tvabYj8mWtSjk2@?fG!D_>lugswKFmzS6_f9rGe&m4`KdvNr&{iENrA~yuX z5FXLjNXj}w97I!XT)*wBp+=@vh%WOj$N>fEk{Wep+eQ=iy%;2ic5y`rkAW%tc~g}+ z(1AP++ylO`&Y;^TDpyqCLUDHCsA~sy(yeb^F6}cgNzN6LQ?s>y5kRZZH9dG>X8KeL zKVNow-D?LF6C8tC4T2b2Lzmin9+#^63P3ShC$RL0oiR8_5E>vuPeO2$&dvY>q7+J` z*=k{OqwZ^iVeA;Gng=gtXh^!LfCVx+;7}(t7T7VN<8!1?A@I)x+ie;GoCO3&3J@q3?ySl7ys?%~mBXHX8GOP`O6}uHssJCQRceVGW$JVfRbC@?<5`gg z4T-y1*>*9(q8Pr=Wo%mh(sQ$u{baiFYWKP*&%+=`m!zDrka4h0-rXD<*NKk;<=SWl zGF$$$=4_@UJ?%6Jrb!n}t!%W=Q>>y%kvT0Uv3YPNx!YREVh^3kIBmeWalfWhYI3~C z(2OFy7$nh)?J>EMtSA$a@7lZe5!?vRI6HJ=UocoX`~!43Y^%LTqqOS7=B-d{dc#y? zs%|i+k$kAw6TH!tRpTM%%hT_Oo@da>PQ85-iZeQtuecJJk>nNgVeF$#f9bXC7Y2l# z<;>n`M-Q4QHjw28G(*(pnrDDfs=h6GtM3I)orOb=?;?N9_R~$jU9+n-MGj7Z|Ewz} zo~WJiP6ZHWH3Q@G*F`b(pX&Bdo%5h)e(^4VpGk$Qn~I9QoohofOnX)?8LjuDMaZesu%66lq7eKS{l$c6 z&e@zKTYc8_EJ}m@Pa-DD`Ef!czHT_tzav)s3Xi?BzQ~M-ibXLyT><|qe}TiOBLnoG zKgKuEol!Q@8xkYow)1k;lh?jY(80oLwRkXwbnuXU`_Qw>w9pW?7_u=#hR0#JB4$1Z zT!md6jePcTLX`haspd&hT}uIs?T9i(b1=T(O8dx^FB>>QcSlXwEi5UaQ`U0^V-KVk z3&e?rOYbj7#n5`ruZ@kY38}X$pbGLLI-+6Hf<>AC-2s5m8|)~M#XkKiMIN;u;Zks< zAOx$C3tb(-V8Z#LecGaq&zFU~{r?AYiGEj(cZ2Kkhc9yv+h1hWe5m^6!Vob-xy;;2 zprW0usO_gAAxg<8v%vWe4BpfuK!JNJM{?gEli_)4`k7FoF5vOw>*uGQq7MW#=DeRY zUk%S<`uyZz0Zr=JcfNjR9S0Khu)>A~gWqpB{WPfBW@938N7|d$(Ea5czeGDjZOUOp zH zgh|eX`d~5N!v?Ul)ysq87*ciFn$@UPT8#34crFN1L=#1ePhVHBq|!d~`PsuP^IkW2 zyRYy2&=BTxd;B-@W{}2%SUAKDAvgWV0jntUOii*WMUtrh9aX=Mm0%j0&fyAqT)wu&@DSj1v(o;fq`jQzlY5DUqask3&KP$xjSFWv407$1}8gY2EO| z+lT^pa9YtcFYeM`S16@p5{(8)`<|Q_XXSrQWe^S+w1n+2qGIatT8*p~{5?Tn(@4-l z5TsVfLXt54(FUhOE9Jq%-LO7hX~!D#8!^Phs2wbn)?owx{hc|2`ud%u#`A;PzGn`| zYXPw*>KTrHG82O(r@CdGNmiY6Nloaucr@`b#m z9IBsDp)*1-qozx47y3X#eQKdP7*8aT$iW$V4p8^e^Eb1UYoyBt`p=?Tq;Ayp5E==p z?FFJ9zQ?qLl#h0$#I_4ZK$}4X8=HCmqYRmz6wx<+&U_8GM564_u<7tE%$hl3%$O5c zwkn`KL5{7czGg^UyzXzvU1#xy{HboF))XfB^k?bLQ;F~^_lBtf#v~(_qLW4W3W>48 zBo;*!yXI_9sHzF7mL?m3}@GA@2BTY5)KL literal 0 HcmV?d00001 diff --git a/workers/package-lock.json b/workers/package-lock.json index 7b7eecb2..efa55497 100644 --- a/workers/package-lock.json +++ b/workers/package-lock.json @@ -19,12 +19,22 @@ "defer-to-connect": "^1.0.1" } }, + "@tokenizer/token": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.1.1.tgz", + "integrity": "sha512-XO6INPbZCxdprl+9qa/AAbFFOMzzwqYxpjPgLICrMD6C2FCw6qfJOPcBk6JqqPLSaZ/Qx87qn4rpPmPMwaAK6w==" + }, "@types/color-name": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", "dev": true }, + "@types/debug": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", + "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" + }, "@types/node": { "version": "13.13.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.5.tgz", @@ -57,6 +67,11 @@ "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.1.1.tgz", "integrity": "sha512-wdlPY2tm/9XBr7QkKlq0WQVgiuGTX6YWPyRyBviSoScBuLfTVQhvwg6wJ369GJ/1nPfTLMfnrFIfjqVg6d+jQQ==" }, + "adm-zip": { + "version": "0.4.14", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.14.tgz", + "integrity": "sha512-/9aQCnQHF+0IiCl0qhXoK7qs//SwYE7zX8lsr/DNk1BRAHYxeLZPL4pguwK29gUEqasYQjqPtEpDRSWEkdHn9g==" + }, "amdefine": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", @@ -1073,6 +1088,17 @@ "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==" }, + "file-type": { + "version": "14.3.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-14.3.0.tgz", + "integrity": "sha512-s71v6jMkbfwVdj87csLeNpL5K93mv4lN+lzgzifoICtPHhnXokDwBa3jrzfg+z6FK872iYJ0vS0i74v8XmoFDA==", + "requires": { + "readable-web-to-node-stream": "^2.0.0", + "strtok3": "^6.0.0", + "token-types": "^2.0.0", + "typedarray-to-buffer": "^3.1.5" + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -1372,6 +1398,11 @@ "safer-buffer": ">= 2.1.2 < 3" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore-by-default": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", @@ -1582,8 +1613,7 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", - "dev": true + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "is-yarn-global": { "version": "0.3.0", @@ -2345,6 +2375,11 @@ "svg-to-pdfkit": "^0.1.8" } }, + "peek-readable": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-3.1.0.tgz", + "integrity": "sha512-KGuODSTV6hcgdZvDrIDBUkN0utcAVj1LL7FfGbM0viKTtCHmtZcuEJ+lGqsp0fTFkGqesdtemV2yUSMeyy3ddA==" + }, "picomatch": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", @@ -2448,6 +2483,11 @@ "util-deprecate": "~1.0.1" } }, + "readable-web-to-node-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-2.0.0.tgz", + "integrity": "sha512-+oZJurc4hXpaaqsN68GoZGQAQIA3qr09Or4fqEsargABnbe5Aau8hFn6ISVleT3cpY/0n/8drn7huyyEvTbghA==" + }, "readdirp": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", @@ -2811,6 +2851,32 @@ "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "dev": true }, + "strtok3": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.0.0.tgz", + "integrity": "sha512-ZXlmE22LZnIBvEU3n/kZGdh770fYFie65u5+2hLK9s74DoFtpkQIdBZVeYEzlolpGa+52G5IkzjUWn+iXynOEQ==", + "requires": { + "@tokenizer/token": "^0.1.1", + "@types/debug": "^4.1.5", + "debug": "^4.1.1", + "peek-readable": "^3.1.0" + }, + "dependencies": { + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -2881,6 +2947,15 @@ "is-number": "^7.0.0" } }, + "token-types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-2.0.0.tgz", + "integrity": "sha512-WWvu8sGK8/ZmGusekZJJ5NM6rRVTTDO7/bahz4NGiSDb/XsmdYBn6a1N/bymUHuWYTWeuLUg98wUzvE4jPdCZw==", + "requires": { + "@tokenizer/token": "^0.1.0", + "ieee754": "^1.1.13" + } + }, "touch": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz", @@ -2923,7 +2998,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", - "dev": true, "requires": { "is-typedarray": "^1.0.0" } diff --git a/workers/package.json b/workers/package.json index af3d05e7..d32b4012 100644 --- a/workers/package.json +++ b/workers/package.json @@ -7,11 +7,13 @@ "author": "Romain Gilliotte", "license": "ISC", "dependencies": { + "adm-zip": "^0.4.14", "aruco-marker": "^2.0.0", "axios": "^0.19.2", "bull": "^3.13.0", "dotenv": "^8.2.0", "excel4node": "^1.7.2", + "file-type": "^14.3.0", "form-data": "^3.0.0", "gm": "^1.23.1", "js-aruco": "^0.1.0", diff --git a/workers/src/helpers/thumbnail.js b/workers/src/helpers/thumbnail.js index d07ab1ce..18fef1d8 100644 --- a/workers/src/helpers/thumbnail.js +++ b/workers/src/helpers/thumbnail.js @@ -48,8 +48,11 @@ async function generateThumbnail(buffer, mimeType, targetRatio = 1.5) { buffer = await util.promisify(image.toBuffer.bind(image))('PNG'); mimeType = 'image/png'; + } else if (mimeType === 'application/zip') { + buffer = fs.readFileSync('data/thumbnails/zipfile.png'); + mimeType = 'image/png'; } else { - buffer = fs.readFileSync('data/placeholder.png'); + buffer = fs.readFileSync('data/thumbnails/placeholder.png'); mimeType = 'image/png'; } diff --git a/workers/src/tasks/uploads/archive/process.js b/workers/src/tasks/uploads/archive/process.js deleted file mode 100644 index 8ba5f424..00000000 --- a/workers/src/tasks/uploads/archive/process.js +++ /dev/null @@ -1,11 +0,0 @@ -const { InputOutput } = require('../../../io'); - -/** - * @param {InputOutput} io - * @param {any} upload - */ -function processArchive(io, upload) { - // Unzip and dispatch -} - -module.exports = { processArchive }; diff --git a/workers/src/tasks/uploads/image/process.js b/workers/src/tasks/uploads/image/process.js index 24b6d9d7..86f1b19f 100644 --- a/workers/src/tasks/uploads/image/process.js +++ b/workers/src/tasks/uploads/image/process.js @@ -12,85 +12,74 @@ const MAX_SIZE = 2560; * @param {any} upload */ async function processImageUpload(io, upload) { - const update = { $set: {} }; let original = cv.imdecode(upload.original.data.buffer, cv.IMREAD_COLOR); - // Start by computing thumbnail - try { - update.$set.thumbnail = await getThumbnail(original); - } catch (e) { - // This is not critical + // Resize source image if too big. This hurts feature detection, but otherwise it takes ages. + if (MAX_SIZE < original.sizes[0] || MAX_SIZE < original.sizes[1]) { + const scale = Math.min(MAX_SIZE / original.sizes[0], MAX_SIZE / original.sizes[1]); + const sizes = original.sizes.map(l => Math.floor(l * scale)); + original = await original.resizeAsync(sizes[0], sizes[1], 0, 0, cv.INTER_CUBIC); } - try { - // Resize source image if too big. This hurts feature detection, but otherwise it takes ages. - if (MAX_SIZE < original.sizes[0] || MAX_SIZE < original.sizes[1]) { - const scale = Math.min(MAX_SIZE / original.sizes[0], MAX_SIZE / original.sizes[1]); - const sizes = original.sizes.map(l => Math.floor(l * scale)); - original = await original.resizeAsync(sizes[0], sizes[1], 0, 0, cv.INTER_CUBIC); - } + // Find reference from the QR code. + const [qrLandmarks, data] = await findQrCode(original); + const [templateId, pageNo] = [data.slice(0, 6), data[6]]; + const template = await io.database.collection('forms').findOne({ randomId: templateId }); + if (!template) { + throw Error('Could not find associated form'); + } - // Find reference from the QR code. - const [qrLandmarks, data] = await findQrCode(original); - const [templateId, pageNo] = [data.slice(0, 6), data[6]]; - const template = await io.database.collection('forms').findOne({ randomId: templateId }); - if (!template) { - throw Error('Could not find associated form'); - } + // Depending on file orientation, chose final size of our image (50px/cm is ~ 125dpi). + let width, height; + if (template.orientation === 'portrait') { + [width, height] = [21.0 * 50, 29.7 * 50]; + } else { + [width, height] = [29.7 * 50, 21.0 * 50]; + } - // Depending on file orientation, chose final size of our image (50px/cm is ~ 125dpi). - let width, height; - if (template.orientation === 'portrait') { - [width, height] = [21.0 * 50, 29.7 * 50]; - } else { - [width, height] = [29.7 * 50, 21.0 * 50]; - } + // Compute regions from template + const regions = {}; + for (let r in template.boundaries) { + const { x, y, w, h, pageNo: boundaryPageNo } = template.boundaries[r]; + if (r === 'corner' || r === 'qr' || r.startsWith('aruco')) continue; + if (Number.isFinite(boundaryPageNo) && pageNo !== boundaryPageNo) continue; - // Compute regions from template - const regions = {}; - for (let r in template.boundaries) { - const { x, y, w, h, pageNo: boundaryPageNo } = template.boundaries[r]; - if (r === 'corner' || r === 'qr' || r.startsWith('aruco')) continue; - if (Number.isFinite(boundaryPageNo) && pageNo !== boundaryPageNo) continue; + regions[r] = { x: x * width, y: y * height, w: w * width, h: h * height }; + } - regions[r] = { x: x * width, y: y * height, w: w * width, h: h * height }; + // Find points in common + const landmarksObj = await findLandmarks(original, qrLandmarks); + const targetObj = computeTargets(template, pageNo, width, height); + const landmarks = []; + const target = []; + for (let key in targetObj) { + if (landmarksObj[key]) { + landmarks.push(landmarksObj[key]); + target.push(targetObj[key]); } + } - // Find points in common - const landmarksObj = await findLandmarks(original, qrLandmarks); - const targetObj = computeTargets(template, pageNo, width, height); - const landmarks = []; - const target = []; - for (let key in targetObj) { - if (landmarksObj[key]) { - landmarks.push(landmarksObj[key]); - target.push(targetObj[key]); - } - } + // Reproject and hope for the best. + const homography = cv.findHomography(landmarks, target); + const document = await original.warpPerspectiveAsync( + homography.homography, + new cv.Size(width, height) + ); - // Reproject and hope for the best. - const homography = cv.findHomography(landmarks, target); - const document = await original.warpPerspectiveAsync( - homography.homography, - new cv.Size(width, height) - ); - - const jpeg = await cv.imencodeAsync('.jpg', document, [cv.IMWRITE_JPEG_QUALITY, 60]); - - update.$set.status = 'pending_dataentry'; - update.$set.dataSourceId = template.dataSourceId; - update.$set.reprojected = { - size: jpeg.byteLength, - mimeType: 'image/jpeg', - data: jpeg, - regions, - }; - } catch (e) { - update.$set.status = 'failed'; - update.$set.reason = e.message; - } + const jpeg = await cv.imencodeAsync('.jpg', document, [cv.IMWRITE_JPEG_QUALITY, 60]); - return update; + return { + $set: { + status: 'pending_dataentry', + dataSourceId: template.dataSourceId, + reprojected: { + size: jpeg.byteLength, + mimeType: 'image/jpeg', + data: jpeg, + regions, + }, + }, + }; } async function findLandmarks(image, qrLandmarks) { diff --git a/workers/src/tasks/uploads/index.js b/workers/src/tasks/uploads/index.js index 6ad256d6..e6ce16ad 100644 --- a/workers/src/tasks/uploads/index.js +++ b/workers/src/tasks/uploads/index.js @@ -2,6 +2,8 @@ const { ObjectId } = require('mongodb'); const { InputOutput } = require('../../io'); const { processImageUpload } = require('./image/process'); const { processPdfUpload } = require('./pdf/process'); +const { processZipUpload } = require('./zip/process'); +const { generateThumbnail } = require('../../helpers/thumbnail'); /** * @param {InputOutput} io @@ -21,23 +23,30 @@ async function processUpload(io, uploadId) { const collection = io.database.collection('input_upload'); const upload = await collection.findOne({ _id: new ObjectId(uploadId) }); + // Process file let update; try { - if (upload.original.mimeType.startsWith('image/')) { + const mimeType = upload.original.mimeType; + if (mimeType.startsWith('image/')) { update = await processImageUpload(io, upload); - } else if (upload.original.mimeType.startsWith('application/pdf')) { + } else if (mimeType.startsWith('application/pdf')) { update = await processPdfUpload(io, upload); - } else throw new Error('Unsupported'); + } else if (mimeType == 'application/zip') { + update = await processZipUpload(io, upload); + } else { + throw new Error('Unsupported'); + } } catch (e) { console.log(e); update = { $set: { status: 'failed', reason: e.message } }; } - if (update) { - await collection.updateOne({ _id: upload._id }, update); - } else { - await collection.deleteOne({ _id: upload._id }); - } + // Add thumbnail + const thumbPng = await generateThumbnail(upload.original.data.buffer, upload.original.mimeType); + update.$set.thumbnail = { size: thumbPng.byteLength, mimeType: 'image/png', data: thumbPng }; + + // Update + await collection.updateOne({ _id: upload._id }, update); } module.exports = { initUploads }; diff --git a/workers/src/tasks/uploads/pdf/process.js b/workers/src/tasks/uploads/pdf/process.js index 5a49d9f2..ea683cc8 100644 --- a/workers/src/tasks/uploads/pdf/process.js +++ b/workers/src/tasks/uploads/pdf/process.js @@ -11,25 +11,40 @@ const { InputOutput } = require('../../../io'); * @param {any} upload */ async function processPdfUpload(io, upload) { - const collection = io.database.collection('input_upload'); - const pdf = gm(upload.original.data.buffer, 'file.pdf'); const identify = promisify(pdf.identify.bind(pdf)); const toBuffer = promisify(pdf.toBuffer.bind(pdf)); const information = await identify(); - const numPages = Array.isArray(information.Format) ? information.Format.length : 1; - for (let i = 0; i < numPages; ++i) { + // No more than 25 pages per PDF. + const numPages = Math.min( + 25, + Array.isArray(information.Format) ? information.Format.length : 1 + ); + + for (let i = numPages - 1; i >= 0; --i) { pdf.selectFrame(i).in('-density', '200'); - const buffer = await toBuffer('JPG'); - const insertion = await collection.insertOne({ + await queueJpg( + io, + upload.projectId, + `${upload.original.name.slice(0, -4)} - page ${i + 1}.jpg`, + await toBuffer('JPG') + ); + } + + return { $set: { status: 'done' } }; +} + +async function queueJpg(io, projectId, filename, buffer) { + try { + const insertion = await io.database.collection('input_upload').insertOne({ status: 'pending_processing', - projectId: upload.projectId, + projectId: projectId, original: { sha1: new Hash('sha1').update(buffer).digest(), - name: `${upload.original.name} [${i + 1}]`, + name: filename, size: buffer.byteLength, mimeType: 'image/jpeg', data: buffer, @@ -41,9 +56,11 @@ async function processPdfUpload(io, upload) { { uploadId: insertion.insertedId }, { attempts: 1, removeOnComplete: true } ); + } catch (e) { + if (!e.message.includes('duplicate key error')) { + throw e; + } } - - return null; } module.exports = { processPdfUpload }; diff --git a/workers/src/tasks/uploads/zip/process.js b/workers/src/tasks/uploads/zip/process.js new file mode 100644 index 00000000..8255f3e3 --- /dev/null +++ b/workers/src/tasks/uploads/zip/process.js @@ -0,0 +1,52 @@ +const AdmZip = require('adm-zip'); +const { Hash } = require('crypto'); +const FileType = require('file-type'); +const { InputOutput } = require('../../../io'); + +/** + * @param {InputOutput} io + * @param {any} upload + */ +async function processZipUpload(io, upload) { + const zip = new AdmZip(upload.original.data.buffer); + for (let entry of zip.getEntries()) { + const buffer = entry.getData(); + + console.log(buffer); + // FIXME: protect against zip bombs! + + await queueFile(io, upload.projectId, entry.name, buffer); + } + + return { $set: { status: 'done' } }; +} + +async function queueFile(io, projectId, filename, buffer) { + const { ext, mime } = await FileType.fromBuffer(buffer); + + try { + const insertion = await io.database.collection('input_upload').insertOne({ + status: 'pending_processing', + projectId: projectId, + original: { + sha1: new Hash('sha1').update(buffer).digest(), + name: filename, + size: buffer.byteLength, + mimeType: mime, + data: buffer, + }, + }); + + await io.queue.add( + 'process-upload', + { uploadId: insertion.insertedId }, + { attempts: 1, removeOnComplete: true } + ); + } catch (e) { + if (!e.message.includes('duplicate key error')) { + throw e; + } + } +} + +module.exports = { processZipUpload };