From f68c0d4ebe91f70299cab11de4e16888949ef6b5 Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Fri, 13 Apr 2018 14:26:21 +0100 Subject: [PATCH 1/9] fix: Use local file when importing csvs --- lib/services/importPersonas/getCsvHeaders.js | 12 ++++++------ lib/services/importPersonas/persistPersonas.js | 2 +- lib/services/importPersonas/uploadPersonas.js | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/services/importPersonas/getCsvHeaders.js b/lib/services/importPersonas/getCsvHeaders.js index a28f038972..6e96c3afe7 100644 --- a/lib/services/importPersonas/getCsvHeaders.js +++ b/lib/services/importPersonas/getCsvHeaders.js @@ -4,8 +4,8 @@ import { isUndefined, uniq } from 'lodash'; import EmptyCsvError from 'lib/errors/EmptyCsvError'; import DuplicateCsvHeadersError from 'lib/errors/DuplicateCsvHeadersError'; -const getCsvHeaders = async (handle) => { - const csvStream = csv.parse({ +const getCsvHeaders = async (fileStream) => { + const csvStreamHandler = csv.parse({ headers: false, quoteHeaders: true }); @@ -13,17 +13,17 @@ const getCsvHeaders = async (handle) => { // read the first row. let headers; - csvStream.on('data', (data) => { + csvStreamHandler.on('data', (data) => { // we're just interested in the first one. if (isUndefined(headers)) headers = data; }); const csvPromise = new Promise((resolve, reject) => { - csvStream.on('error', reject); - csvStream.on('finish', resolve); + csvStreamHandler.on('error', reject); + csvStreamHandler.on('finish', resolve); }); - fileService.downloadToStream(handle)(csvStream); + fileStream.pipe(csvStreamHandler); await csvPromise; diff --git a/lib/services/importPersonas/persistPersonas.js b/lib/services/importPersonas/persistPersonas.js index 69a01f99a7..5778df50b7 100644 --- a/lib/services/importPersonas/persistPersonas.js +++ b/lib/services/importPersonas/persistPersonas.js @@ -2,7 +2,7 @@ import fs from 'fs'; import clamscan from 'lib/services/files/clamscan'; import { uploadFromStream } from 'lib/services/files/storage'; -export const PERSONAS_CSV_PATH = '/personasCsvs'; +export const PERSONAS_CSV_PATH = 'personasCsvs'; export default async ({ file, diff --git a/lib/services/importPersonas/uploadPersonas.js b/lib/services/importPersonas/uploadPersonas.js index e08da78e2b..456f698116 100644 --- a/lib/services/importPersonas/uploadPersonas.js +++ b/lib/services/importPersonas/uploadPersonas.js @@ -1,3 +1,4 @@ +import fs from 'fs'; import persistPersonas from 'lib/services/importPersonas/persistPersonas'; import PersonasImport from 'lib/models/personasImport'; import { STAGE_CONFIGURE_FIELDS } from 'lib/constants/personasImport'; @@ -29,7 +30,10 @@ const uploadPersonas = async ({ id }); - const csvHeaders = await getCsvHeaders(handle); + // instead of going back out to s3, use the local file we already have! + const filePath = file.path; + const fileStream = fs.createReadStream(filePath); + const csvHeaders = await getCsvHeaders(fileStream); const structure = await getStructure({ csvHeaders, From c5d1d58ec979b7b556247589fa061b4335017f08 Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Fri, 13 Apr 2018 16:05:19 +0100 Subject: [PATCH 2/9] ci(lint): Remove unused import --- lib/services/importPersonas/getCsvHeaders.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/services/importPersonas/getCsvHeaders.js b/lib/services/importPersonas/getCsvHeaders.js index 6e96c3afe7..79ab19df2b 100644 --- a/lib/services/importPersonas/getCsvHeaders.js +++ b/lib/services/importPersonas/getCsvHeaders.js @@ -1,4 +1,3 @@ -import * as fileService from 'lib/services/files'; import csv from 'fast-csv'; import { isUndefined, uniq } from 'lodash'; import EmptyCsvError from 'lib/errors/EmptyCsvError'; From f613af4f0aaeb2bd4ccaa7293a30a3baef6e8fc0 Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Fri, 13 Apr 2018 17:13:31 +0100 Subject: [PATCH 3/9] fix: Use clamdscan for faster scanning --- .env.example | 3 ++- lib/services/files/clamscan.js | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index ad0891651a..41d74d0361 100644 --- a/.env.example +++ b/.env.example @@ -193,4 +193,5 @@ FS_REPO=local ######## # Location of virus scanning binary (ClamAV - https://www.clamav.net/) -#CLAMSCAN_BINARY=/usr/bin/clamscan +#CLAMDSCAN_BINARY=/usr/bin/clamscan +#CLAMDSCAN_CONF=/etc/clamav/clam.conf diff --git a/lib/services/files/clamscan.js b/lib/services/files/clamscan.js index dfa8aac75e..d2e314ff98 100644 --- a/lib/services/files/clamscan.js +++ b/lib/services/files/clamscan.js @@ -8,17 +8,18 @@ const virusError = () => ( new Error('This file has not passed the virus scan and will be deleted.') ); export default filePath => new Promise((resolve, reject) => { - if (!process.env.CLAMSCAN_BINARY) { - logger.warn('CLAMSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.'); + if (!process.env.CLAMDSCAN_BINARY || !process.env.CLAMDSCAN_CONF) { + logger.warn('CLAMDSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.'); return resolve(filePath); } try { const clam = require('clamscan')({ remove_infected: true, - clamscan: { - path: process.env.CLAMSCAN_BINARY, + clamdscan: { + path: process.env.CLAMDSCAN_BINARY, + conf: process.env.CLAMDSCAN_CONF, }, - preference: 'clamscan', + preference: 'clamdscan', }); clam.is_infected(filePath, (err, file, isInfected) => { From bd7bbe99d24370d984e2095dc4cd470a54362d15 Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Mon, 16 Apr 2018 12:43:33 +0100 Subject: [PATCH 4/9] Update .env.example --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 41d74d0361..f089e95423 100644 --- a/.env.example +++ b/.env.example @@ -193,5 +193,5 @@ FS_REPO=local ######## # Location of virus scanning binary (ClamAV - https://www.clamav.net/) -#CLAMDSCAN_BINARY=/usr/bin/clamscan +#CLAMDSCAN_BINARY=/usr/bin/clamdscan #CLAMDSCAN_CONF=/etc/clamav/clam.conf From 489a37b86d24df222d376156c7ddb3d74356b3af Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Mon, 16 Apr 2018 12:46:20 +0100 Subject: [PATCH 5/9] Update .env.example --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index f089e95423..3ad6fe79aa 100644 --- a/.env.example +++ b/.env.example @@ -194,4 +194,4 @@ FS_REPO=local # Location of virus scanning binary (ClamAV - https://www.clamav.net/) #CLAMDSCAN_BINARY=/usr/bin/clamdscan -#CLAMDSCAN_CONF=/etc/clamav/clam.conf +#CLAMDSCAN_CONF=/etc/clamav/clamd.conf From 4ad1a173cdce2ef55957003e75cd2612616c9eb3 Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Tue, 17 Apr 2018 09:09:57 +0100 Subject: [PATCH 6/9] fix: Handle clamscan failures greacefully --- lib/services/files/clamscan.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/services/files/clamscan.js b/lib/services/files/clamscan.js index d2e314ff98..2e46e50f50 100644 --- a/lib/services/files/clamscan.js +++ b/lib/services/files/clamscan.js @@ -1,33 +1,36 @@ import Promise from 'bluebird'; import logger from 'lib/logger'; +import defaultTo from 'lodash/defaultTo'; -const scanError = err => ( - new Error(`Something went wrong when scanning the file. ${err.message}`) -); const virusError = () => ( new Error('This file has not passed the virus scan and will be deleted.') ); export default filePath => new Promise((resolve, reject) => { - if (!process.env.CLAMDSCAN_BINARY || !process.env.CLAMDSCAN_CONF) { + if (!process.env.CLAMDSCAN_BINARY) { logger.warn('CLAMDSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.'); return resolve(filePath); } try { + const config_file = defaultTo(process.env.CLAMDSCAN_CONF, '/etc/clamav/clamd.conf'); const clam = require('clamscan')({ remove_infected: true, clamdscan: { path: process.env.CLAMDSCAN_BINARY, - conf: process.env.CLAMDSCAN_CONF, + config_file }, preference: 'clamdscan', }); clam.is_infected(filePath, (err, file, isInfected) => { - if (err) return reject(scanError(err)); + if (err) { + logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err); + return resolve(filePath); + } if (isInfected) return reject(virusError()); return resolve(filePath); }); } catch (err) { - return reject(scanError(err)); + logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err); + return resolve(filePath); } }); From 16f9aa9c16de57c2d861aa070127618e018b676a Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Tue, 17 Apr 2018 10:54:18 +0100 Subject: [PATCH 7/9] ci(lint): Linting fixes --- lib/models/statementForwarding.js | 4 ++-- lib/services/files/clamscan.js | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/models/statementForwarding.js b/lib/models/statementForwarding.js index b78826f220..6a2dfacb04 100644 --- a/lib/models/statementForwarding.js +++ b/lib/models/statementForwarding.js @@ -166,7 +166,7 @@ const schema = new mongoose.Schema({ query: { type: String, validate: { - validator: value => { + validator: (value) => { try { JSON.parse(value); } catch (err) { @@ -174,7 +174,7 @@ const schema = new mongoose.Schema({ } return true; }, - message: "Invalid query" + message: 'Invalid query' } }, isPublic: { type: Boolean, default: false } diff --git a/lib/services/files/clamscan.js b/lib/services/files/clamscan.js index 2e46e50f50..23fb58ba7d 100644 --- a/lib/services/files/clamscan.js +++ b/lib/services/files/clamscan.js @@ -11,12 +11,11 @@ export default filePath => new Promise((resolve, reject) => { return resolve(filePath); } try { - const config_file = defaultTo(process.env.CLAMDSCAN_CONF, '/etc/clamav/clamd.conf'); const clam = require('clamscan')({ remove_infected: true, clamdscan: { path: process.env.CLAMDSCAN_BINARY, - config_file + config_file: defaultTo(process.env.CLAMDSCAN_CONF, '/etc/clamav/clamd.conf') }, preference: 'clamdscan', }); From 1ecddb3fa3386a449d42008982f548bc6a7319f5 Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Tue, 17 Apr 2018 12:37:25 +0100 Subject: [PATCH 8/9] fix: Show persona dates --- lib/helpers/getDateFromMongoID.js | 15 ++++++++++ lib/helpers/tests/getDateFromMongoID-test.js | 26 ++++++++++++++++ ui/src/containers/Owner/CreatedAt.js | 6 ++-- ui/src/containers/Owner/index.js | 31 +++++++++++++++----- 4 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 lib/helpers/getDateFromMongoID.js create mode 100644 lib/helpers/tests/getDateFromMongoID-test.js diff --git a/lib/helpers/getDateFromMongoID.js b/lib/helpers/getDateFromMongoID.js new file mode 100644 index 0000000000..165c4cf7d2 --- /dev/null +++ b/lib/helpers/getDateFromMongoID.js @@ -0,0 +1,15 @@ +import isString from 'lodash/isString'; + +export default _id => { + if (!isString(_id) || _id.length != 24) { + // not a mongo id + throw Error('Not a Mongo ID'); + } + const timehex = _id.substring(0,8); + + // convert to a number... base 16 + const secondsSinceEpoch = parseInt(timehex, 16); + + // convert to milliseconds, and create a new date + return new Date(secondsSinceEpoch*1000); +} \ No newline at end of file diff --git a/lib/helpers/tests/getDateFromMongoID-test.js b/lib/helpers/tests/getDateFromMongoID-test.js new file mode 100644 index 0000000000..1aaac5a09f --- /dev/null +++ b/lib/helpers/tests/getDateFromMongoID-test.js @@ -0,0 +1,26 @@ +import { expect } from 'chai'; +import getDateFromMongoID from '../getDateFromMongoID'; + +const _id = '5ad52af00000000000000000'; +describe.only('getDateFromMongoID', () => { + it('should return date for Mongo ID as string', () => { + const date = getDateFromMongoID(_id); + const expectedDate = new Date("2018-04-16T23:00:00"); + expect(date.getTime()).to.equal(expectedDate.getTime()); + }); + + it('should error when _id is not a string', () => { + expect(getDateFromMongoID.bind(null, 123)).to.throw(Error); + expect(getDateFromMongoID.bind(null, false)).to.throw(Error); + expect(getDateFromMongoID.bind(null, undefined)).to.throw(Error); + expect(getDateFromMongoID.bind(null, null)).to.throw(Error); + expect(getDateFromMongoID.bind(null, {})).to.throw(Error); + expect(getDateFromMongoID.bind(null, [])).to.throw(Error); + }); + + it('should error when _id is not 24 chars', () => { + expect(getDateFromMongoID.bind(null, 'abc')).to.throw(Error); + expect(getDateFromMongoID.bind(null, `${_id}0`)).to.throw(Error); + }); + +}); diff --git a/ui/src/containers/Owner/CreatedAt.js b/ui/src/containers/Owner/CreatedAt.js index 5ddf2e284e..2fc7e8e082 100644 --- a/ui/src/containers/Owner/CreatedAt.js +++ b/ui/src/containers/Owner/CreatedAt.js @@ -1,4 +1,6 @@ import moment from 'moment'; -export default createdAt => - `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`; +export default createdAt => { + if (!createdAt) return ''; + return `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`; +} diff --git a/ui/src/containers/Owner/index.js b/ui/src/containers/Owner/index.js index cfd0bdca9c..d2770a74d4 100644 --- a/ui/src/containers/Owner/index.js +++ b/ui/src/containers/Owner/index.js @@ -1,24 +1,41 @@ import React from 'react'; +import getDateFromMongoID from 'lib/helpers/getDateFromMongoID'; import { withProps, compose } from 'recompose'; import { withModel } from 'ui/utils/hocs'; import createdAtFormatter from './CreatedAt'; +const getCreatedDate = (createdAt, _id) => { + if (!createdAt && _id) { + try { + return getDateFromMongoID(_id); + } catch (err) { + // error getting date from id (might not be a valid mongo id) + return; + } + } + return createdAt; +} + const Creator = compose( withProps(({ model }) => ({ schema: 'user', id: model.get('owner') })), withModel -)(({ model, createdAt }) => { +)(({ model, date }) => { const email = model.get('email', false); const name = model.get('name', false); return (
Made by { email ? {name || email} : deleted user }
-
{ createdAtFormatter(createdAt) }
+
{ date }
); }); -export default ({ model }) => ( - model.has('owner') - ? - :
{ createdAtFormatter(model.get('createdAt')) }
-); +export default ({ model }) => { + const date = getCreatedDate(model.get('createdAt', false), model.get('_id')); + const formattedDate = createdAtFormatter(date); + return ( + model.has('owner') + ? + :
{ formattedDate }
+ ) +}; From 82c7b402e148007db3a1d363cdb3f246ffcd428e Mon Sep 17 00:00:00 2001 From: James Mullaney Date: Tue, 17 Apr 2018 13:07:54 +0100 Subject: [PATCH 9/9] ci(lint): Fix linting --- lib/helpers/getDateFromMongoID.js | 15 +++++++++------ lib/helpers/tests/getDateFromMongoID-test.js | 7 +++++-- ui/src/containers/Owner/CreatedAt.js | 4 ++-- ui/src/containers/Owner/index.js | 6 +++--- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/lib/helpers/getDateFromMongoID.js b/lib/helpers/getDateFromMongoID.js index 165c4cf7d2..0d62447e22 100644 --- a/lib/helpers/getDateFromMongoID.js +++ b/lib/helpers/getDateFromMongoID.js @@ -1,15 +1,18 @@ -import isString from 'lodash/isString'; +import isString from 'lodash/isString'; -export default _id => { - if (!isString(_id) || _id.length != 24) { +export default (_id) => { + if (!isString(_id) || _id.length !== 24) { // not a mongo id throw Error('Not a Mongo ID'); } - const timehex = _id.substring(0,8); + const timehex = _id.substring(0, 8); // convert to a number... base 16 const secondsSinceEpoch = parseInt(timehex, 16); + if (isNaN(secondsSinceEpoch)) { + throw Error('Invalid date in Mongo ID'); + } // convert to milliseconds, and create a new date - return new Date(secondsSinceEpoch*1000); -} \ No newline at end of file + return new Date(secondsSinceEpoch * 1000); +}; diff --git a/lib/helpers/tests/getDateFromMongoID-test.js b/lib/helpers/tests/getDateFromMongoID-test.js index 1aaac5a09f..2b1e6be9ea 100644 --- a/lib/helpers/tests/getDateFromMongoID-test.js +++ b/lib/helpers/tests/getDateFromMongoID-test.js @@ -2,10 +2,10 @@ import { expect } from 'chai'; import getDateFromMongoID from '../getDateFromMongoID'; const _id = '5ad52af00000000000000000'; -describe.only('getDateFromMongoID', () => { +describe('getDateFromMongoID', () => { it('should return date for Mongo ID as string', () => { const date = getDateFromMongoID(_id); - const expectedDate = new Date("2018-04-16T23:00:00"); + const expectedDate = new Date('2018-04-16T23:00:00'); expect(date.getTime()).to.equal(expectedDate.getTime()); }); @@ -23,4 +23,7 @@ describe.only('getDateFromMongoID', () => { expect(getDateFromMongoID.bind(null, `${_id}0`)).to.throw(Error); }); + it('should error when _id does not contain a valid hex date', () => { + expect(getDateFromMongoID.bind(null, 'Zad52af00000000000000000')).to.throw(Error); + }); }); diff --git a/ui/src/containers/Owner/CreatedAt.js b/ui/src/containers/Owner/CreatedAt.js index 2fc7e8e082..2e30382717 100644 --- a/ui/src/containers/Owner/CreatedAt.js +++ b/ui/src/containers/Owner/CreatedAt.js @@ -1,6 +1,6 @@ import moment from 'moment'; -export default createdAt => { +export default (createdAt) => { if (!createdAt) return ''; return `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`; -} +}; diff --git a/ui/src/containers/Owner/index.js b/ui/src/containers/Owner/index.js index d2770a74d4..fedaa690a2 100644 --- a/ui/src/containers/Owner/index.js +++ b/ui/src/containers/Owner/index.js @@ -14,7 +14,7 @@ const getCreatedDate = (createdAt, _id) => { } } return createdAt; -} +}; const Creator = compose( withProps(({ model }) => ({ schema: 'user', id: model.get('owner') })), @@ -35,7 +35,7 @@ export default ({ model }) => { const formattedDate = createdAtFormatter(date); return ( model.has('owner') - ? + ? :
{ formattedDate }
- ) + ); };