Skip to content

Commit

Permalink
more work on file uploads
Browse files Browse the repository at this point in the history
  • Loading branch information
@romain-gilliotte
romain-gilliotte committed May 13, 2020
1 parent b6f3fd5 commit a76ab43
Show file tree
Hide file tree
Showing 28 changed files with 736 additions and 323 deletions.
32 changes: 20 additions & 12 deletions api/src/io.js
View file
Expand Up @@ -6,29 +6,37 @@ const Redlock = require('redlock');

class InputOutput {
async connect() {
this.mongo = await MongoClient.connect(config.mongo.uri, { useUnifiedTopology: true });

this.database = this.mongo.db(config.mongo.database);
this.database
.collection('invitation')
.createIndex({ projectId: 1, email: 1 }, { unique: true });
this.database.collection('input').createIndex({ sequenceId: 1, 'content.variableId': 1 });
this.database.collection('user').createIndex({ subs: 1 });

// FIXME move this somewhere else
this.mongo = await MongoClient.connect(config.mongo.uri, {
useUnifiedTopology: true,
poolSize: 50,
});

this.redis = new Redis(config.redis.uri);
this.redisLock = new Redlock([this.redis]);
this.queue = new Bull('workers', config.redis.uri);

this._createDatabase();
}

async disconnect() {
this.mongo.close(true);

this.queue.close();

this.redis.disconnect();
}

_createDatabase() {
this.database = this.mongo.db(config.mongo.database);
this.database
.collection('invitation')
.createIndex({ projectId: 1, email: 1 }, { unique: true });

this.database
.collection('input_upload')
.createIndex({ 'original.sha1': 1 }, { unique: true });

this.database.collection('input').createIndex({ sequenceId: 1, 'content.variableId': 1 });
this.database.collection('user').createIndex({ subs: 1 });
}
}

module.exports = { InputOutput };
114 changes: 87 additions & 27 deletions api/src/routers/uploads.js
View file
Expand Up @@ -5,19 +5,65 @@ const Router = require('@koa/router');
const multer = require('@koa/multer');
const { ObjectId } = require('mongodb');
const JSONStream = require('JSONStream');
const { Transform, pipeline } = require('stream');

const router = new Router();

router.get('/project/:id/upload', async ctx => {
const forms = await ctx.io.database
.collection('input_upload')
.find(
{ projectId: new ObjectId(ctx.params.id) },
{ projection: { 'original.data': 0, 'reprojected.data': 0 } }
const collection = ctx.io.database.collection('input_upload');

if (ctx.request.accepts('application/json')) {
const filter = { projectId: new ObjectId(ctx.params.id) };
const projection = { 'original.data': 0, 'thumbnail.data': 0, 'reprojected.data': 0 };
const forms = collection.find(
{ ...filter, status: { $ne: 'done' } },
{ projection, sort: [['_id', -1]] }
);

ctx.response.type = 'application/json';
ctx.response.body = forms.pipe(JSONStream.stringify());
ctx.response.type = 'application/json';
ctx.response.body = forms.pipe(JSONStream.stringify());
} else if (ctx.request.accepts('text/event-stream')) {
const options = { batchSize: 1, fullDocument: 'updateLookup' };
const wpipeline = [
{ $match: { 'fullDocument.projectId': new ObjectId(ctx.params.id) } },
{
$project: {
'fullDocument.original.data': 0,
'fullDocument.thumbnail.data': 0,
'fullDocument.reprojected.data': 0,
'updateDescription.updatedFields.thumbnail.data': 0,
'updateDescription.updatedFields.reprojected.data': 0,
},
},
];

const changeLog = collection.watch(wpipeline, options);
const transform = new Transform({
objectMode: true,
highWaterMark: 1,
transform: (chunk, encoding, callback) => {
if (['insert', 'update'].includes(chunk.operationType)) {
let action = { type: chunk.operationType, id: chunk.documentKey._id };

if (action.type === 'insert') {
action.document = chunk.fullDocument;
} else if (action.type === 'update') {
action.update = chunk.updateDescription.updatedFields;
}

callback(null, `data: ${JSON.stringify(action)}\n\n`);
}
},
});

// Close changelog on all errors (most notably, client disconnect when leaving the page).
pipeline(changeLog, transform, error => void changeLog.close());

ctx.response.type = 'text/event-stream';
ctx.response.body = transform;
} else {
ctx.response.status = 406;
}
});

router.get('/project/:projectId/upload/:id', async ctx => {
Expand All @@ -41,32 +87,35 @@ router.get('/project/:projectId/upload/:id/:name(original|reprojected|thumbnail)
if (upload[ctx.params.name]) {
ctx.response.type = upload[ctx.params.name].mimeType;
ctx.response.body = upload[ctx.params.name].data.buffer;
} else if (ctx.params.name === 'thumbnail') {
ctx.response.type = 'image/png';
ctx.response.body = await promisify(readFile)('data/placeholder.png');
}
});

router.post('/project/:projectId/upload', multer().single('file'), async ctx => {
const file = ctx.request.file;

const insertion = await ctx.io.database.collection('input_upload').insertOne({
status: 'pending_processing',
projectId: new ObjectId(ctx.params.projectId),
original: {
sha1: new Hash('sha1').update(file.buffer).digest(),
name: file.originalname,
size: file.size,
mimeType: file.mimetype,
data: file.buffer,
},
});
try {
const insertion = await ctx.io.database.collection('input_upload').insertOne({
status: 'pending_processing',
projectId: new ObjectId(ctx.params.projectId),
original: {
sha1: new Hash('sha1').update(file.buffer).digest(),
name: file.originalname,
size: file.size,
mimeType: file.mimetype,
data: file.buffer,
},
});

await ctx.io.queue.add(
'process-upload',
{ uploadId: insertion.insertedId },
{ attempts: 1, removeOnComplete: true }
);
await ctx.io.queue.add(
'process-upload',
{ uploadId: insertion.insertedId },
{ attempts: 1, removeOnComplete: true }
);
} catch (e) {
if (!e.message.includes('duplicate key error')) {
throw e;
}
}

ctx.response.status = 204;
});
Expand All @@ -77,8 +126,19 @@ router.post('/project/:projectId/upload', multer().single('file'), async ctx =>
// _id: new ObjectId(ctx.params.id),
// projectId: new ObjectId(ctx.params.projectId),
// },
// { $set: { inputId: ctx.body.inputId } }
// { $set: { status: 'done' } }
// );

// ctx.response.status = 204;
// });

router.delete('/project/:projectId/upload/:id', async ctx => {
await ctx.io.database.collection('input_upload').deleteOne({
_id: new ObjectId(ctx.params.id),
projectId: new ObjectId(ctx.params.projectId),
});

ctx.response.status = 204;
});

module.exports = router;
3 changes: 2 additions & 1 deletion docker-compose.yml
View file
Expand Up @@ -10,12 +10,13 @@ services:
MONGO_INITDB_ROOT_PASSWORD: admin
volumes:
- mongo_data:/data/db
command: --replSet rs0

redis:
image: redis
ports:
- "6379:6379"

unoconv:
image: zrrrzzt/docker-unoconv-webservice
ports:
Expand Down
34 changes: 29 additions & 5 deletions frontend/src/components/pages/input-edition/project-input-edition.html
View file
Expand Up @@ -4,10 +4,33 @@
<legend translate="project.general_informations"></legend>

<div class="form-group">
<label class="col-sm-2 control-label" translate="project.collection_form"></label>
<div class="col-sm-10 metadata">
<div class="input-group" style="width: 100%;">
<select
class="form-control"
ng-options="ds.id as ds.name for ds in $ctrl.project.forms"
ng-model="$ctrl.dataSourceId"
ng-disabled="!$ctrl.dataSourceEditable"
></select>
<span class="input-group-btn">
<button
ng-disabled="!$ctrl.dataSourceId || !$ctrl.dataSourceEditable"
ng-click="$ctrl.onDataSourceSet()"
class="btn btn-default"
>
Valider la source
</button>
</span>
</div>
</div>
</div>

<div class="form-group" ng-if="!$ctrl.dataSourceEditable">
<label class="col-sm-2 control-label" translate="project.collection_site"></label>
<div class="col-sm-10 metadata">
<image-scroll
ng-if="$ctrl.upload"
ng-if="$ctrl.upload.reprojected.regions.site"
upload="$ctrl.upload"
region="site"
></image-scroll>
Expand All @@ -21,14 +44,15 @@
</div>
</div>

<div class="form-group">
<div class="form-group" ng-if="!$ctrl.dataSourceEditable">
<label class="col-sm-2 control-label" translate="project.covered_period"></label>
<div class="col-sm-10 metadata">
<image-scroll
ng-if="$ctrl.upload"
ng-if="$ctrl.upload.reprojected.regions.period"
upload="$ctrl.upload"
region="period"
></image-scroll>

<select
class="form-control"
ng-options="slot as slot|formatSlot for slot in $ctrl.periods"
Expand All @@ -38,7 +62,7 @@
</div>
</div>

<div class="form-group">
<div class="form-group" ng-if="$ctrl.upload">
<label class="col-sm-2 control-label" translate="project.original_file"></label>
<div class="col-sm-10 metadata">
<a
Expand Down Expand Up @@ -93,7 +117,7 @@

<div class="col-sm-10">
<image-scroll
ng-if="$ctrl.upload"
ng-if="$ctrl.upload && $ctrl.upload.reprojected.regions[variable.id]"
class="img-thumbnail"
upload="$ctrl.upload"
region="{{variable.id}}"
Expand Down
60 changes: 29 additions & 31 deletions frontend/src/components/pages/input-edition/project-input-edition.js
View file
Expand Up @@ -28,14 +28,7 @@ module.config($stateProvider => {
resolve: {
period: $stateParams => $stateParams.period,
siteId: $stateParams => $stateParams.siteId,
variables: ($stateParams, project) => {
const ds = project.forms.find(f => f.id === $stateParams.dataSourceId);
return ds.elements.filter(v => ds.active && v.active);
},
periodicity: ($stateParams, project) =>
project.forms.find(f => f.id === $stateParams.dataSourceId).periodicity,
entityIds: ($stateParams, project) =>
project.forms.find(f => f.id === $stateParams.dataSourceId).entities,
dataSourceId: $stateParams => $stateParams.dataSourceId,
},
});

Expand All @@ -49,22 +42,9 @@ module.config($stateProvider => {
.get(`/project/${$stateParams.projectId}/upload/${$stateParams.uploadId}`)
.then(response => response.data),

dataSourceId: upload => upload.dataSourceId,
period: () => null,
siteId: () => null,
variables: (project, upload) => {
const ds = project.forms.find(f => f.id === upload.dataSourceId);
return ds.elements.filter(
v => ds.active && v.active && upload.reprojected.regions[v.id]
);
},
periodicity: (project, upload) => {
const ds = project.forms.find(f => f.id === upload.dataSourceId);
return ds.periodicity;
},
entityIds: (project, upload) => {
const ds = project.forms.find(f => f.id === upload.dataSourceId);
return ds.entities;
},
},
});
});
Expand All @@ -75,6 +55,7 @@ module.component(__componentName, {
invitations: '<',

upload: '<',
dataSourceId: '<',
period: '<',
siteId: '<',
variables: '<',
Expand Down Expand Up @@ -118,17 +99,34 @@ module.component(__componentName, {
}

$onChanges(changes) {
// Compute choices.
this._initChoices();
this.dataSourceEditable = true;
this.metadataEditable = true;

this.onDataSourceSet();
this.onMetadataSet();
}

// If metadata is already set, we load the file.
if (this.period && this.siteId) {
this.metadataEditable = false;
this.onMetadataSet();
} else {
this.period = TimeSlot.fromDate(new Date(), this.periodicity).value;
this.metadataEditable = true;
async onDataSourceSet() {
if (!this.dataSourceId) {
return;
}

this.dataSourceEditable = false;

const dataSource = this.project.forms.find(f => f.id === this.dataSourceId);
this.periodicity = dataSource.periodicity;
this.entityIds = dataSource.entities;
this.variables = dataSource.elements.filter(v => {
if (this.upload && this.upload.reprojected) {
return v.active && this.upload.reprojected.regions[v.id];
} else {
return v.active;
}
});

// Compute choices.
this._initChoices();
this.period = TimeSlot.fromDate(new Date(), this.periodicity).value;
}

async onMetadataSet() {
Expand Down
22 changes: 22 additions & 0 deletions frontend/src/components/pages/input-uploads/dropzone.html
View file
@@ -0,0 +1,22 @@
<div
ng-on-dragover="$ctrl.onDragOver($event)"
ng-on-dragleave="$ctrl.onDragLeave($event)"
ng-on-drop="$ctrl.onDrop($event)"
ng-class="{highlight: $ctrl.highlight}"
>
<p>
Déposez ici vos fiches de saisies remplies ou
<label for="file-input">choisissez des fichier</label>
<input
type="file"
id="file-input"
accept="{{$ctrl.inputTypes}}"
multiple
ng-on-change="$ctrl.onInputChange($event)"
/>
</p>
<p style="font-size: 70%;">
Formats supportés: Photos, scans ou fax (pdf, jpg, png, tiff), excel (xlsx) ou archives
(zip).
</p>
</div>

0 comments on commit a76ab43

Please sign in to comment.