LL-522 Use local file when importing csvs, and use clamd

.env.example

@@ -193,4 +193,5 @@ FS_REPO=local
 ########
 
 # Location of virus scanning binary (ClamAV - https://www.clamav.net/)
-#CLAMSCAN_BINARY=/usr/bin/clamscan
+#CLAMDSCAN_BINARY=/usr/bin/clamdscan
+#CLAMDSCAN_CONF=/etc/clamav/clamd.conf

lib/helpers/getDateFromMongoID.js

@@ -0,0 +1,18 @@
+import isString from 'lodash/isString';
+
+export default (_id) => {
+  if (!isString(_id) || _id.length !== 24) {
+    // not a mongo id
+    throw Error('Not a Mongo ID');
+  }
+  const timehex = _id.substring(0, 8);
+
+  // convert to a number... base 16
+  const secondsSinceEpoch = parseInt(timehex, 16);
+  if (isNaN(secondsSinceEpoch)) {
+    throw Error('Invalid date in Mongo ID');
+  }
+
+  // convert to milliseconds, and create a new date
+  return new Date(secondsSinceEpoch * 1000);
+};

lib/helpers/tests/getDateFromMongoID-test.js

@@ -0,0 +1,29 @@
+import { expect } from 'chai';
+import getDateFromMongoID from '../getDateFromMongoID';
+
+const _id = '5ad52af00000000000000000';
+describe('getDateFromMongoID', () => {
+  it('should return date for Mongo ID as string', () => {
+    const date = getDateFromMongoID(_id);
+    const expectedDate = new Date('2018-04-16T23:00:00');
+    expect(date.getTime()).to.equal(expectedDate.getTime());
+  });
+
+  it('should error when _id is not a string', () => {
+    expect(getDateFromMongoID.bind(null, 123)).to.throw(Error);
+    expect(getDateFromMongoID.bind(null, false)).to.throw(Error);
+    expect(getDateFromMongoID.bind(null, undefined)).to.throw(Error);
+    expect(getDateFromMongoID.bind(null, null)).to.throw(Error);
+    expect(getDateFromMongoID.bind(null, {})).to.throw(Error);
+    expect(getDateFromMongoID.bind(null, [])).to.throw(Error);
+  });
+
+  it('should error when _id is not 24 chars', () => {
+    expect(getDateFromMongoID.bind(null, 'abc')).to.throw(Error);
+    expect(getDateFromMongoID.bind(null, `${_id}0`)).to.throw(Error);
+  });
+
+  it('should error when _id does not contain a valid hex date', () => {
+    expect(getDateFromMongoID.bind(null, 'Zad52af00000000000000000')).to.throw(Error);
+  });
+});

lib/models/statementForwarding.js

@@ -166,15 +166,15 @@ const schema = new mongoose.Schema({
   query: {
     type: String,
     validate: {
-      validator: value => {
+      validator: (value) => {
         try {
           JSON.parse(value);
         } catch (err) {
           return false;
         }
         return true;
       },
-      message: "Invalid query"
+      message: 'Invalid query'
     }
   },
   isPublic: { type: Boolean, default: false }

lib/services/files/clamscan.js

@@ -1,32 +1,35 @@
 import Promise from 'bluebird';
 import logger from 'lib/logger';
+import defaultTo from 'lodash/defaultTo';
 
-const scanError = err => (
-  new Error(`Something went wrong when scanning the file. ${err.message}`)
-);
 const virusError = () => (
   new Error('This file has not passed the virus scan and will be deleted.')
 );
 export default filePath => new Promise((resolve, reject) => {
-  if (!process.env.CLAMSCAN_BINARY) {
-    logger.warn('CLAMSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.');
+  if (!process.env.CLAMDSCAN_BINARY) {
+    logger.warn('CLAMDSCAN NOT INSTALLED, SEE DOCS FOR FURTHER INFORMATION.');
     return resolve(filePath);
   }
   try {
     const clam = require('clamscan')({
       remove_infected: true,
-      clamscan: {
-        path: process.env.CLAMSCAN_BINARY,
+      clamdscan: {
+        path: process.env.CLAMDSCAN_BINARY,
+        config_file: defaultTo(process.env.CLAMDSCAN_CONF, '/etc/clamav/clamd.conf')
       },
-      preference: 'clamscan',
+      preference: 'clamdscan',
     });
 
     clam.is_infected(filePath, (err, file, isInfected) => {
-      if (err) return reject(scanError(err));
+      if (err) {
+        logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err);
+        return resolve(filePath);
+      }
       if (isInfected) return reject(virusError());
       return resolve(filePath);
     });
   } catch (err) {
-    return reject(scanError(err));
+    logger.warn('ERROR SCANNING FILE WITH CLAMD - ', err);
+    return resolve(filePath);
   }
 });

lib/services/importPersonas/getCsvHeaders.js

@@ -1,29 +1,28 @@
-import * as fileService from 'lib/services/files';
 import csv from 'fast-csv';
 import { isUndefined, uniq } from 'lodash';
 import EmptyCsvError from 'lib/errors/EmptyCsvError';
 import DuplicateCsvHeadersError from 'lib/errors/DuplicateCsvHeadersError';
 
-const getCsvHeaders = async (handle) => {
-  const csvStream = csv.parse({
+const getCsvHeaders = async (fileStream) => {
+  const csvStreamHandler = csv.parse({
     headers: false,
     quoteHeaders: true
   });
 
   // read the first row.
   let headers;
 
-  csvStream.on('data', (data) => {
+  csvStreamHandler.on('data', (data) => {
     // we're just interested in the first one.
     if (isUndefined(headers)) headers = data;
   });
 
   const csvPromise = new Promise((resolve, reject) => {
-    csvStream.on('error', reject);
-    csvStream.on('finish', resolve);
+    csvStreamHandler.on('error', reject);
+    csvStreamHandler.on('finish', resolve);
   });
 
-  fileService.downloadToStream(handle)(csvStream);
+  fileStream.pipe(csvStreamHandler);
 
   await csvPromise;
 

lib/services/importPersonas/persistPersonas.js

@@ -2,7 +2,7 @@ import fs from 'fs';
 import clamscan from 'lib/services/files/clamscan';
 import { uploadFromStream } from 'lib/services/files/storage';
 
-export const PERSONAS_CSV_PATH = '/personasCsvs';
+export const PERSONAS_CSV_PATH = 'personasCsvs';
 
 export default async ({
   file,

lib/services/importPersonas/uploadPersonas.js

@@ -1,3 +1,4 @@
+import fs from 'fs';
 import persistPersonas from 'lib/services/importPersonas/persistPersonas';
 import PersonasImport from 'lib/models/personasImport';
 import { STAGE_CONFIGURE_FIELDS } from 'lib/constants/personasImport';
@@ -29,7 +30,10 @@ const uploadPersonas = async ({
     id
   });
 
-  const csvHeaders = await getCsvHeaders(handle);
+  // instead of going back out to s3, use the local file we already have!
+  const filePath = file.path;
+  const fileStream = fs.createReadStream(filePath);
+  const csvHeaders = await getCsvHeaders(fileStream);
 
   const structure = await getStructure({
     csvHeaders,

ui/src/containers/Owner/CreatedAt.js

@@ -1,4 +1,6 @@
 import moment from 'moment';
 
-export default createdAt =>
-  `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`;
+export default (createdAt) => {
+  if (!createdAt) return '';
+  return `Created ${moment(createdAt).fromNow()} - ${moment(createdAt).format('YYYY-MM-DD HH:mm:ss')}`;
+};

ui/src/containers/Owner/index.js

@@ -1,24 +1,41 @@
 import React from 'react';
+import getDateFromMongoID from 'lib/helpers/getDateFromMongoID';
 import { withProps, compose } from 'recompose';
 import { withModel } from 'ui/utils/hocs';
 import createdAtFormatter from './CreatedAt';
 
+const getCreatedDate = (createdAt, _id) => {
+  if (!createdAt && _id) {
+    try {
+      return getDateFromMongoID(_id);
+    } catch (err) {
+      // error getting date from id (might not be a valid mongo id)
+      return;
+    }
+  }
+  return createdAt;
+};
+
 const Creator = compose(
   withProps(({ model }) => ({ schema: 'user', id: model.get('owner') })),
   withModel
-)(({ model, createdAt }) => {
+)(({ model, date }) => {
   const email = model.get('email', false);
   const name = model.get('name', false);
   return (
     <div>
       <div style={{ fontWeight: 'bold' }}>Made by { email ? <i>{name || email}</i> : <i>deleted user</i> }</div>
-      <div>{ createdAtFormatter(createdAt) }</div>
+      <div>{ date }</div>
     </div>
   );
 });
 
-export default ({ model }) => (
-  model.has('owner')
-  ? <Creator model={model} createdAt={model.get('createdAt', false)} />
-  : <div style={{ marginTop: 8 }}>{ createdAtFormatter(model.get('createdAt')) }</div>
-);
+export default ({ model }) => {
+  const date = getCreatedDate(model.get('createdAt', false), model.get('_id'));
+  const formattedDate = createdAtFormatter(date);
+  return (
+    model.has('owner')
+    ? <Creator model={model} date={formattedDate} />
+    : <div style={{ marginTop: 8 }}>{ formattedDate }</div>
+  );
+};