From fc479c0a1b700154f5bc6cb376711e977495d3ab Mon Sep 17 00:00:00 2001 From: Hugo Freire Date: Sat, 27 May 2017 17:19:05 +0200 Subject: [PATCH] chore: import last existing code from sandbox --- package.json | 9 +- src/auth/facebook.js | 107 +++++++ src/auth/index.js | 10 + src/auth/preload.js | 88 ++++++ src/database.js | 118 -------- src/database/auth.js | 64 +++++ src/database/index.js | 12 + src/database/people.js | 92 ++++++ src/database/sqlite.js | 91 ++++++ src/providers/errors/index.js | 10 + src/providers/errors/not-authorized-error.js | 22 ++ src/providers/tinder.js | 92 +++++- src/routes/app/index.html | 13 +- src/routes/people.js | 2 +- src/routes/train.js | 20 +- src/server.js | 77 ++--- src/taste.js | 146 +++++++--- src/utils/rekognition.js | 69 +++-- src/utils/s3.js | 73 +++-- test/utils/s3.js | 280 +++++++++++++++++++ 20 files changed, 1114 insertions(+), 281 deletions(-) create mode 100644 src/auth/facebook.js create mode 100644 src/auth/index.js create mode 100644 src/auth/preload.js delete mode 100644 src/database.js create mode 100644 src/database/auth.js create mode 100644 src/database/index.js create mode 100644 src/database/people.js create mode 100644 src/database/sqlite.js create mode 100644 src/providers/errors/index.js create mode 100644 src/providers/errors/not-authorized-error.js create mode 100644 test/utils/s3.js diff --git a/package.json b/package.json index 582def026..5a2d5e763 100644 --- a/package.json +++ b/package.json @@ -15,13 +15,18 @@ "dependencies": { "aws-sdk": "2.58.0", "bluebird": "3.5.0", + "bluebird-retry": "0.10.1", + "brakes": "2.5.3", + "health-checkup": "1.0.8", "lodash": "4.17.4", "modern-logger": "1.3.7", + "nightmare": "2.10.0", + "random-http-useragent": "1.1.0", "request": "2.81.0", "serverful": "1.0.22", - "sqlite3": "^3.1.8", + "sqlite3": "3.1.8", "tinder": "1.19.0", - "uuid": "^3.0.1" + "uuid": "3.0.1" }, "devDependencies": { "babel-eslint": "7.2.3", diff --git a/src/auth/facebook.js b/src/auth/facebook.js new file mode 100644 index 000000000..37f85bded --- /dev/null +++ b/src/auth/facebook.js @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +const _ = require('lodash') +const Promise = require('bluebird') +const retry = require('bluebird-retry') +const Brakes = require('brakes') + +const Nightmare = require('nightmare') +Nightmare.Promise = Promise + +const Logger = require('modern-logger') + +const Health = require('health-checkup') + +const RandomUserAgent = require('random-http-useragent') + +const { join } = require('path') + +const authorizeApp = function (email, password, url, userAgent) { + let facebookUserId + let accessToken + + const nightmare = Nightmare(this._options.nightmare) + + return nightmare + .useragent(userAgent) + .on('page', function (type, url, method, response) { + if (type !== 'xhr-complete') { + return + } + + if (url.path === '/ajax/presence/reconnect.php') { + const match = response.match(/"user_channel":"p_(.*?)"/) + facebookUserId = match.length === 2 ? match[ 1 ] : undefined + + return + } + + if (_.includes(url, 'oauth/confirm?dpr')) { + const match = response.match(/access_token=(.*)&/) + accessToken = match.length === 2 ? match[ 1 ] : undefined + } + }) + .goto('https://facebook.com') + .type('input#email', email) + .type('input#pass', password) + .click('#loginbutton input') + .wait(3000) + .goto(url) + .wait('button._42ft._4jy0.layerConfirm._1flv._51_n.autofocus.uiOverlayButton._4jy5._4jy1.selected._51sy') + .click('button._42ft._4jy0.layerConfirm._1flv._51_n.autofocus.uiOverlayButton._4jy5._4jy1.selected._51sy') + .wait(10000) + .end() + .then(() => { + if (!accessToken || !facebookUserId) { + throw new Error('unable to authorize app') + } + + return { accessToken, facebookUserId } + }) +} + +const defaultOptions = { + nightmare: { + show: false, + partition: 'nopersist', + webPreferences: { + preload: join(__dirname, '/preload.js'), + webSecurity: false + } + }, + retry: { max_tries: 2, interval: 5000, timeout: 40000, throw_original: true }, + breaker: { timeout: 60000, threshold: 80, circuitDuration: 3 * 60 * 60 * 1000 } +} + +class Facebook { + constructor (options = {}) { + this._options = _.defaults(options, defaultOptions) + + this._breaker = new Brakes(this._options.breaker) + + this._authorizeAppCircuitBreaker = this._breaker.slaveCircuit((...params) => retry(() => authorizeApp.bind(this)(...params), this._options.retry)) + + Health.addCheck('facebook', () => new Promise((resolve, reject) => { + if (this._breaker.isOpen()) { + return reject(new Error(`circuit breaker is open`)) + } else { + return resolve() + } + })) + } + + authorizeApp (email, password, url) { + Logger.debug('Started authorizing app in Facebook') + + return RandomUserAgent.get() + .then((userAgent) => this._authorizeAppCircuitBreaker.exec(email, password, url, userAgent)) + .finally(() => Logger.debug('Finished authorizing app in Facebook')) + } +} + +module.exports = Facebook diff --git a/src/auth/index.js b/src/auth/index.js new file mode 100644 index 000000000..8eb1a38b1 --- /dev/null +++ b/src/auth/index.js @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +module.exports = { + Facebook: require('./facebook') +} diff --git a/src/auth/preload.js b/src/auth/preload.js new file mode 100644 index 000000000..7eac432f8 --- /dev/null +++ b/src/auth/preload.js @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +/* eslint-disable no-undef */ + +window.__nightmare = {} +__nightmare.ipc = require('electron').ipcRenderer +__nightmare.sliced = require('sliced') + +// Listen for error events +window.addEventListener('error', function (e) { + __nightmare.ipc.send('page', 'error', e.message, e.error.stack) +}) + +var open = window.XMLHttpRequest.prototype.open +window.XMLHttpRequest.prototype.open = function (method, url, async, user, pass) { + this.addEventListener('readystatechange', function () { + if (this.readyState === 4) { + __nightmare.ipc.send('page', 'xhr-complete', url, method, this.responseText) + } + }, false) + open.apply(this, arguments) +}; + +(function () { + // prevent 'unload' and 'beforeunload' from being bound + var defaultAddEventListener = window.addEventListener + window.addEventListener = function (type) { + if (type === 'unload' || type === 'beforeunload') { + return + } + defaultAddEventListener.apply(window, arguments) + } + + // prevent 'onunload' and 'onbeforeunload' from being set + Object.defineProperties(window, { + onunload: { + enumerable: true, + writable: false, + value: null + }, + onbeforeunload: { + enumerable: true, + writable: false, + value: null + } + }) + + // listen for console.log + var defaultLog = console.log + console.log = function () { + __nightmare.ipc.send('console', 'log', __nightmare.sliced(arguments)) + return defaultLog.apply(this, arguments) + } + + // listen for console.warn + var defaultWarn = console.warn + console.warn = function () { + __nightmare.ipc.send('console', 'warn', __nightmare.sliced(arguments)) + return defaultWarn.apply(this, arguments) + } + + // listen for console.error + var defaultError = console.error + console.error = function () { + __nightmare.ipc.send('console', 'error', __nightmare.sliced(arguments)) + return defaultError.apply(this, arguments) + } + + // overwrite the default alert + window.alert = function (message) { + __nightmare.ipc.send('page', 'alert', message) + } + + // overwrite the default prompt + window.prompt = function (message, defaultResponse) { + __nightmare.ipc.send('page', 'prompt', message, defaultResponse) + } + + // overwrite the default confirm + window.confirm = function (message, defaultResponse) { + __nightmare.ipc.send('page', 'confirm', message, defaultResponse) + } +})() diff --git a/src/database.js b/src/database.js deleted file mode 100644 index 8a1b1644d..000000000 --- a/src/database.js +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2017, Hugo Freire . - * - * This source code is licensed under the license found in the - * LICENSE file in the root directory of this source tree. - */ - -const Promise = require('bluebird') - -const sqlite3 = require('sqlite3') - -const { mkdirAsync, existsSync } = Promise.promisifyAll(require('fs')) - -const { join } = require('path') - -const createFile = function () { - return Promise.resolve() - .then(() => { - const path = join(__dirname, '../tmp/') - - if (!existsSync(path)) { - return mkdirAsync(path) - } - }) - .then(() => { - return new Promise((resolve, reject) => { - const path = join(__dirname, '../tmp/get-me-a-tinder-date.db') - const options = sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE - const callback = (error) => { - if (error) { - reject(error) - - return - } - - resolve() - } - - this._database = Promise.promisifyAll(new sqlite3.Database(path, options, callback)) - }) - }) -} - -const createSchema = function () { - return this._database.runAsync( - 'CREATE TABLE IF NOT EXISTS people (' + - 'id VARCHAR(36), ' + - 'created_date DATETIME DEFAULT CURRENT_TIMESTAMP, ' + - 'updated_date DATETIME DEFAULT CURRENT_TIMESTAMP, ' + - 'like INTEGER NOT NULL DEFAULT 0,' + - 'train INTEGER NOT NULL DEFAULT 0,' + - 'provider VARCHAR(32), ' + - 'provider_id VARCHAR(64), ' + - 'data TEXT,' + - 'PRIMARY KEY (provider, provider_id)' + - ')') -} - -const queryAllPeople = function (query) { - return this._database.allAsync(query) - .mapSeries((row) => { - row.created_date = new Date(row.created_date) - row.data = JSON.parse(row.data) - return row - }) -} - -class Database { - start () { - if (this._database) { - return Promise.resolve() - } - - return createFile.bind(this)() - .then(() => createSchema.bind(this)()) - } - - savePeople (id, like, train, provider, providerId, data) { - return Promise.resolve() - .then(() => this.findPeopleById(id)) - .then((person) => { - if (person) { - return this._database.runAsync( - `UPDATE people SET id = ?, updated_date = ?, like = ?, train = ?, provider = ?, provider_id = ?, data = ?`, - [ id, new Date(), like, train, provider, providerId, JSON.stringify(data) ]) - } else { - return this._database.runAsync( - `INSERT INTO people (id, like, train, provider, provider_id, data) VALUES (?, ?, ?, ?, ?, ?)`, - [ id, like, train, provider, providerId, JSON.stringify(data) ]) - .catch((error) => { - if (error.code !== 'SQLITE_CONSTRAINT') { - throw error - } - }) - } - }) - } - - findAllPeople () { - return queryAllPeople.bind(this)('SELECT * FROM people ORDER BY created_date DESC') - } - - findPeopleById (id) { - return this._database.getAsync('SELECT * FROM people WHERE id = ?', [ id ]) - .then((person) => { - if (!person) { - return - } - - person.created_date = new Date(person.created_date) - person.data = JSON.parse(person.data) - - return person - }) - } -} - -module.exports = new Database() diff --git a/src/database/auth.js b/src/database/auth.js new file mode 100644 index 000000000..72e3a7274 --- /dev/null +++ b/src/database/auth.js @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +const _ = require('lodash') + +const SQLite = require('./sqlite') + +class Auth { + save (provider, data) { + const _data = _.clone(data) + + if (_data.created_date instanceof Date) { + _data.created_date = _data.created_date.toISOString().replace(/T/, ' ').replace(/\..+/, '') + } + + if (_data.updated_date instanceof Date) { + _data.updated_date = _data.updated_date.toISOString().replace(/T/, ' ').replace(/\..+/, '') + } + + const keys = _.keys(_data) + const values = _.values(_data) + + return Promise.resolve() + .then(() => this.findByProvider(provider)) + .then((auth) => { + if (auth) { + keys.push('updated_date') + values.push(new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')) + + return SQLite.run(`UPDATE auth SET ${keys.map((key) => `${key} = ?`)} WHERE provider = ?`, values.concat([ provider ])) + } else { + return SQLite.run(`INSERT INTO auth (${keys}) VALUES (${values.map(() => '?')})`, values) + .catch((error) => { + if (error.code !== 'SQLITE_CONSTRAINT') { + throw error + } + }) + } + }) + } + + findByProvider (provider) { + return SQLite.get('SELECT * FROM auth WHERE provider = ?', [ provider ]) + .then((authentication) => { + if (!authentication) { + return + } + + authentication.created_date = new Date(authentication.created_date) + + return authentication + }) + } + + deleteByProvider (provider) { + return SQLite.run('DELETE FROM auth WHERE provider = ?', [ provider ]) + } +} + +module.exports = new Auth() diff --git a/src/database/index.js b/src/database/index.js new file mode 100644 index 000000000..c6952aab8 --- /dev/null +++ b/src/database/index.js @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +module.exports = { + SQLite: require('./sqlite'), + People: require('./people'), + Auth: require('./auth') +} diff --git a/src/database/people.js b/src/database/people.js new file mode 100644 index 000000000..25132f9a1 --- /dev/null +++ b/src/database/people.js @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +const _ = require('lodash') + +const SQLite = require('./sqlite') + +const queryAllPeople = function (query) { + return SQLite.all(query) + .mapSeries((row) => { + row.created_date = new Date(row.created_date) + row.data = JSON.parse(row.data) + return row + }) +} + +class People { + save (provider, providerId, data) { + const _data = _.clone(data) + + if (_data.created_date instanceof Date) { + _data.created_date = _data.created_date.toISOString().replace(/T/, ' ').replace(/\..+/, '') + } + + if (_data.updated_date instanceof Date) { + _data.updated_date = _data.updated_date.toISOString().replace(/T/, ' ').replace(/\..+/, '') + } + + if (_data.data) { + _data.data = JSON.stringify(_data.data) + } + + const keys = _.keys(_data) + const values = _.values(_data) + + return Promise.resolve() + .then(() => this.findByProviderAndProviderId(provider, providerId)) + .then((person) => { + if (person) { + keys.push('updated_date') + values.push(new Date().toISOString().replace(/T/, ' ').replace(/\..+/, '')) + + return SQLite.run(`UPDATE people SET ${keys.map((key) => `${key} = ?`)} WHERE provider = ? AND provider_id = ?`, values.concat([ provider, providerId ])) + } else { + return SQLite.run(`INSERT INTO people (${keys}) VALUES (${values.map(() => '?')})`, values) + .catch((error) => { + if (error.code !== 'SQLITE_CONSTRAINT') { + throw error + } + }) + } + }) + } + + findAll () { + return queryAllPeople.bind(this)('SELECT * FROM people ORDER BY created_date DESC') + } + + findById (id) { + return SQLite.get('SELECT * FROM people WHERE id = ?', [ id ]) + .then((person) => { + if (!person) { + return + } + + person.created_date = new Date(person.created_date) + person.data = JSON.parse(person.data) + + return person + }) + } + + findByProviderAndProviderId (provider, providerId) { + return SQLite.get('SELECT * FROM people WHERE provider = ? AND provider_id = ?', [ provider, providerId ]) + .then((person) => { + if (!person) { + return + } + + person.created_date = new Date(person.created_date) + person.data = JSON.parse(person.data) + + return person + }) + } +} + +module.exports = new People() diff --git a/src/database/sqlite.js b/src/database/sqlite.js new file mode 100644 index 000000000..b79e39999 --- /dev/null +++ b/src/database/sqlite.js @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +const Promise = require('bluebird') + +const sqlite3 = require('sqlite3') + +const { mkdirAsync, existsSync } = Promise.promisifyAll(require('fs')) + +const { join } = require('path') + +const createFile = function () { + return Promise.resolve() + .then(() => { + const path = join(__dirname, '../../tmp/') + + if (!existsSync(path)) { + return mkdirAsync(path) + } + }) + .then(() => { + return new Promise((resolve, reject) => { + const path = join(__dirname, '../../tmp/get-me-a-date.db') + const options = sqlite3.OPEN_READWRITE | sqlite3.OPEN_CREATE + const callback = (error) => { + if (error) { + reject(error) + + return + } + + resolve() + } + + this._database = Promise.promisifyAll(new sqlite3.Database(path, options, callback)) + }) + }) +} + +const createSchema = function () { + return this._database.runAsync( + 'CREATE TABLE IF NOT EXISTS people (' + + 'id VARCHAR(36) NOT NULL, ' + + 'created_date DATETIME DEFAULT CURRENT_TIMESTAMP, ' + + 'updated_date DATETIME DEFAULT CURRENT_TIMESTAMP, ' + + 'like INTEGER NOT NULL DEFAULT 0,' + + 'photos_similarity_mean REAL NOT NULL,' + + 'train INTEGER NOT NULL DEFAULT 0,' + + 'provider VARCHAR(32) NOT NULL, ' + + 'provider_id VARCHAR(64) NOT NULL, ' + + 'data TEXT,' + + 'PRIMARY KEY (provider, provider_id)' + + ')') + .then(() => this._database.runAsync( + 'CREATE TABLE IF NOT EXISTS auth (' + + 'provider VARCHAR(32) NOT NULL, ' + + 'created_date DATETIME DEFAULT CURRENT_TIMESTAMP, ' + + 'updated_date DATETIME DEFAULT CURRENT_TIMESTAMP, ' + + 'api_token TEXT NOT NULL,' + + 'PRIMARY KEY (provider)' + + ')')) +} + +class SQLite { + start () { + if (this._database) { + return Promise.resolve() + } + + return createFile.bind(this)() + .then(() => createSchema.bind(this)()) + } + + run (...args) { + return this._database.runAsync(...args) + } + + get (...args) { + return this._database.getAsync(...args) + } + + all (...args) { + return this._database.allAsync(...args) + } +} + +module.exports = new SQLite() diff --git a/src/providers/errors/index.js b/src/providers/errors/index.js new file mode 100644 index 000000000..0be56ba6e --- /dev/null +++ b/src/providers/errors/index.js @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +module.exports = { + NotAuthorizedError: require('./not-authorized-error') +} diff --git a/src/providers/errors/not-authorized-error.js b/src/providers/errors/not-authorized-error.js new file mode 100644 index 000000000..12420d005 --- /dev/null +++ b/src/providers/errors/not-authorized-error.js @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +class NotAuthorizedError extends Error { + constructor (message) { + super(message) + + this.name = this.constructor.name + + if (typeof Error.captureStackTrace === 'function') { + Error.captureStackTrace(this, this.constructor) + } else { + this.stack = (new Error(message)).stack + } + } +} + +module.exports = NotAuthorizedError diff --git a/src/providers/tinder.js b/src/providers/tinder.js index 8c94f430a..2cf5777be 100644 --- a/src/providers/tinder.js +++ b/src/providers/tinder.js @@ -5,34 +5,96 @@ * LICENSE file in the root directory of this source tree. */ -const FACEBOOK_USER_ID = process.env.FACEBOOK_USER_ID -const FACEBOOK_OAUTH_ACCESS_TOKEN = process.env.FACEBOOK_OAUTH_ACCESS_TOKEN -let TINDER_API_TOKEN = process.env.TINDER_API_TOKEN +const FACEBOOK_USER_EMAIL = process.env.FACEBOOK_USER_EMAIL +const FACEBOOK_USER_PASSWORD = process.env.FACEBOOK_USER_PASSWORD +const FACEBOOK_TINDER_APP_AUTHZ_URL = 'https://www.facebook.com/v2.6/dialog/oauth?redirect_uri=fb464891386855067%3A%2F%2Fauthorize%2F&state=%7B%22challenge%22%3A%22q1WMwhvSfbWHvd8xz5PT6lk6eoA%253D%22%2C%220_auth_logger_id%22%3A%2254783C22-558A-4E54-A1EE-BB9E357CC11F%22%2C%22com.facebook.sdk_client_state%22%3Atrue%2C%223_method%22%3A%22sfvc_auth%22%7D&scope=user_birthday%2Cuser_photos%2Cuser_education_history%2Cemail%2Cuser_relationship_details%2Cuser_friends%2Cuser_work_history%2Cuser_likes&response_type=token%2Csigned_request&default_audience=friends&return_scopes=true&auth_type=rerequest&client_id=464891386855067&ret=login&sdk=ios&logger_id=54783C22-558A-4E54-A1EE-BB9E357CC11F#_=_' +const _ = require('lodash') const Promise = require('bluebird') +const retry = require('bluebird-retry') +const Brakes = require('brakes') + +const Health = require('health-checkup') + +const { NotAuthorizedError } = require('./errors') const { TinderClient } = require('tinder') -class Tinder { - constructor () { - this.client = Promise.promisifyAll(new TinderClient()) +const { Facebook } = require('../auth') +const { Auth } = require('../database') + +const facebookAuthorizeTinderApp = function () { + return this._facebook.authorizeApp(FACEBOOK_USER_EMAIL, FACEBOOK_USER_PASSWORD, FACEBOOK_TINDER_APP_AUTHZ_URL) + .then(({ accessToken, facebookUserId }) => { + return this._tinder.authorizeCircuitBreaker.exec(accessToken, facebookUserId) + .then(() => { + const apiToken = this._tinder.getAuthToken() + + return Auth.save('tinder', { provider: 'tinder', api_token: apiToken }) + }) + }) +} + +const handleError = function (error) { + switch (error.message) { + case 'Unauthorized': + this._tinder.setAuthToken() + + return Auth.deleteByProvider('tinder') + .then(() => { + throw new NotAuthorizedError() + }) + default: + throw error } +} - authenticate () { - if (TINDER_API_TOKEN) { - this.client.setAuthToken(TINDER_API_TOKEN) +const defaultOptions = { + retry: { max_tries: 2, interval: 1000, throw_original: true }, + breaker: { timeout: 5000, threshold: 80, circuitDuration: 3 * 60 * 60 * 1000 } +} + +class Tinder { + constructor (options = {}) { + this._options = _.defaults(options, defaultOptions) + + this._tinder = Promise.promisifyAll(new TinderClient()) + this._facebook = new Facebook() + + this._breaker = new Brakes(this._options.breaker) + + this._tinder.authorizeCircuitBreaker = this._breaker.slaveCircuit((...params) => retry(() => this._tinder.authorizeAsync(...params), this._options.retry)) + this._tinder.getRecommendationsCircuitBreaker = this._breaker.slaveCircuit((params) => retry(() => this._tinder.getRecommendationsAsync(params), this._options.retry)) + + Health.addCheck('tinder', () => new Promise((resolve, reject) => { + if (this._breaker.isOpen()) { + return reject(new Error(`circuit breaker is open`)) + } else { + return resolve() + } + })) + } - return Promise.resolve() - } + authorize () { + return Auth.findByProvider('tinder') + .then((auth) => { + if (!_.has(auth, 'api_token')) { + return facebookAuthorizeTinderApp.bind(this)() + } - return this.client.authorizeAsync(FACEBOOK_OAUTH_ACCESS_TOKEN, FACEBOOK_USER_ID) - .then(() => { - TINDER_API_TOKEN = this.client.getAuthToken() + this._tinder.setAuthToken(auth.api_token) }) } getRecommendations () { - return this.client.getRecommendationsAsync(10) + return Promise.try(() => { + if (!this._tinder.getAuthToken()) { + throw new NotAuthorizedError() + } + }) + .then(() => this._tinder.getRecommendationsCircuitBreaker.exec(10)) + .then(({ results }) => results) + .catch((error) => handleError.bind(this)(error)) } } diff --git a/src/routes/app/index.html b/src/routes/app/index.html index c8be72fd3..9b78ca634 100644 --- a/src/routes/app/index.html +++ b/src/routes/app/index.html @@ -49,6 +49,10 @@ .like:before { background: rgba(0, 255, 0, 0.5); } + + .train:before { + background: rgba(255, 255, 0, 0.5); + } @@ -59,7 +63,10 @@

{{ p.data.schools && p.data.schools[0].name || p.data.jobs && p.data.jobs[0]

{{ p.data.bio }}

  • - +
@@ -68,8 +75,8 @@

{{ p.data.schools && p.data.schools[0].name || p.data.jobs && p.data.jobs[0]
  • diff --git a/src/routes/people.js b/src/routes/people.js index 7bee9ebd1..a30e25135 100644 --- a/src/routes/people.js +++ b/src/routes/people.js @@ -17,7 +17,7 @@ class People extends Route { } handler (request, reply) { - Database.findAllPeople() + Database.People.findAll() .then((people) => reply(null, people)) .catch((error) => { Logger.error(error) diff --git a/src/routes/train.js b/src/routes/train.js index 32943c008..e123ab03b 100644 --- a/src/routes/train.js +++ b/src/routes/train.js @@ -7,15 +7,12 @@ const { Route } = require('serverful') -const _ = require('lodash') -const Promise = require('bluebird') - const Logger = require('modern-logger') const Taste = require('../taste') -const Database = require('../database') +const { People } = require('../database') -class People extends Route { +class Train extends Route { constructor () { super('POST', '/train/{id}', 'People', 'Returns all people') } @@ -23,7 +20,7 @@ class People extends Route { handler (request, reply) { const { id } = request.params - Database.findPeopleById(id) + People.findById(id) .then((person) => { if (!person) { reply(null) @@ -31,12 +28,11 @@ class People extends Route { return } - const urls = _.map(person.data.photos, 'url') + const { provider, provider_id, data } = person + const { photos } = data - return Taste.mentalSnapshot(urls) - .then(() => { - return Database.savePeople(person.id, person.like, true, person.provider, person.providerId, person.data) - }) + return Taste.mentalSnapshot(photos) + .then(() => People.save(provider, provider_id, { train: true })) }) .then(() => reply(null)) .catch((error) => { @@ -51,4 +47,4 @@ class People extends Route { } } -module.exports = new People() +module.exports = new Train() diff --git a/src/server.js b/src/server.js index 78b3a2214..d1a38d365 100644 --- a/src/server.js +++ b/src/server.js @@ -7,77 +7,80 @@ const FIND_DATES_PERIOD = process.env.FIND_DATES_PERIOD || 10 * 60 * 1000 +const { Serverful } = require('serverful') + const _ = require('lodash') const Promise = require('bluebird') -const { Serverful } = require('serverful') - const Logger = require('modern-logger') const Tinder = require('./providers/tinder') +const { NotAuthorizedError } = require('./providers/errors') const Taste = require('./taste') -const Database = require('./database') +const { SQLite, People } = require('./database') const uuidV4 = require('uuid/v4') +const checkRecommendationOut = (provider, rec) => { + const providerId = rec._id + const person = {} + + return Promise.props({ photos: Taste.checkPhotosOut(rec.photos) }) + .then(({ photos }) => { + person.id = uuidV4() + person.provider = provider + person.provider_id = providerId + person.like = photos.like + person.photos_similarity_mean = photos.faceSimilarityMean + person.data = rec + + return People.save(provider, providerId, person) + .then(() => person) + }) +} + const findDates = function () { return Tinder.getRecommendations() - .then(({ results }) => { - Logger.info(`Got ${results.length} recommendations`) - - return Promise.mapSeries(results, (result) => { - const { name, photos, _id } = result - - Logger.info(`Started checking ${name} out with ${photos.length} photos`) + .then((recs) => { + const provider = 'tinder' - return Promise.props({ - photos: Taste.checkPhotosOut(photos) - }) - .then(({ photos }) => { - const meta = { - id: uuidV4(), - provider: 'tinder', - photos: {} - } + Logger.info(`Got ${recs.length} recommendations from ${_.capitalize(provider)}`) - meta.photos.similarity_mean = _.round(_.mean(_.without(photos, 0, undefined)), 2) || 0 - - Logger.info(`${name} face is ${meta.photos.similarity_mean}% similar with target group`) - - meta.like = !_.isEmpty(photos) && meta.photos.similarity_mean > 0 - - result.meta = meta - - return Database.savePeople(meta.id, meta.like, false, meta.provider, _id, result) - .finally(() => Logger.info(`${name} got a ${meta.like ? 'like :+1:' : 'pass :-1:'}`)) + return Promise.map(recs, (rec) => { + return checkRecommendationOut(provider, rec) + .then(({ photos_similarity_mean, data, like }) => { + // eslint-disable-next-line camelcase + return Logger.info(`${data.name} got a ${like ? 'like :+1:' : 'pass :-1:'}(photos = ${photos_similarity_mean}%)`) }) .catch((error) => Logger.warn(error)) - }) + }, { concurrency: 5 }) }) + .catch(NotAuthorizedError, () => Tinder.authorize()) + .catch((error) => Logger.error(error)) } class Server extends Serverful { start () { - return super.start() - .then(() => Database.start()) - .then(() => Taste.start()) - - .then(() => Tinder.authenticate()) + return Promise.all([ super.start(), SQLite.start().then(() => Tinder.authorize()), Taste.bootstrap() ]) .then(() => { if (FIND_DATES_PERIOD > 0) { this.findDates() } }) - } findDates () { + const startDate = _.now() + Logger.info('Started finding dates') return findDates.bind(this)() .finally(() => { - Logger.info('Finished finding dates') + const stopDate = _.now() + const duration = _.round((stopDate - startDate) / 1000, 1) + + Logger.info(`Finished finding dates (time = ${duration}s)`) this.timeout = setTimeout(() => this.findDates(), FIND_DATES_PERIOD) }) diff --git a/src/taste.js b/src/taste.js index a1dcda12f..af63ea8c0 100644 --- a/src/taste.js +++ b/src/taste.js @@ -21,6 +21,29 @@ const Rekognition = require('./utils/rekognition') const request = Promise.promisifyAll(require('request').defaults({ encoding: null })) +const { parse } = require('url') + +const savePhoto = function (photo) { + if (!photo) { + return Promise.reject(new Error('invalid arguments')) + } + + const url = parse(photo.url) + if (!url) { + return Promise.reject(new Error('invalid photo url')) + } + + return request.getAsync(url.href) + .then(({ body }) => { + return this.s3.putObject(`photos/tinder${url.pathname}`, body) + .then(() => { + photo.url = `https://s3-${AWS_REGION}.amazonaws.com/${AWS_S3_BUCKET}/photos/tinder${url.pathname}` + + return body + }) + }) +} + class Taste { constructor () { this.rekognition = new Rekognition({ @@ -37,7 +60,7 @@ class Taste { }) } - start () { + bootstrap () { return this.createRekognitionCollectionIfNeeded() .then(() => this.syncS3BucketAndRekognitionCollection()) } @@ -52,60 +75,109 @@ class Taste { } syncS3BucketAndRekognitionCollection () { + const start = _.now() + return this.rekognition.listFaces(AWS_REKOGNITION_COLLECTION) .then(({ Faces }) => { - const currentFaces = _.map(Faces, 'ExternalImageId') + const currentImages = _(Faces) + .map(({ ExternalImageId }) => ExternalImageId) + .uniq() + .value() return this.s3.listObjects('train') - .then((availableFaces) => { - const facesToDelete = _.difference(currentFaces, availableFaces) - const facesToIndex = _.difference(availableFaces, currentFaces) - - const _facesToDelete = _(facesToDelete) - .filter((externalImageId) => _.find(Faces, { ExternalImageId: externalImageId })) - .map((externalImageId) => _.find(Faces, { ExternalImageId: externalImageId }).FaceId) - .value() - - return Promise.all([ - this.rekognition.deleteFaces(AWS_REKOGNITION_COLLECTION, _facesToDelete) - .catch((error) => Logger.warn(error)), - Promise.mapSeries(facesToIndex, (face) => { - return this.rekognition.indexFaces(AWS_REKOGNITION_COLLECTION, AWS_S3_BUCKET, `train/${face}`) + .then((availableImages) => { + const imagesToDelete = _.difference(currentImages, availableImages) + const imagesToIndex = _.difference(availableImages, currentImages) + + const facesToDelete = [] + _.forEach(imagesToDelete, (externalImageId) => { + const images = _.filter(Faces, { ExternalImageId: externalImageId }) + + _.forEach(images, ({ FaceId }) => { + facesToDelete.push(FaceId) + }) + }) + + let deletedFaces = 0 + const deleteFaces = function () { + return this.rekognition.deleteFaces(AWS_REKOGNITION_COLLECTION, facesToDelete) + .then(() => { + deletedFaces = facesToDelete.length + }) + .catch((error) => Logger.warn(error)) + } + + let indexedFaces = 0 + const indexFaces = function () { + return Promise.map(imagesToIndex, (image) => { + return this.rekognition.indexFaces(AWS_REKOGNITION_COLLECTION, AWS_S3_BUCKET, `${image}`) + .then((data) => { + if (!data.FaceRecords || _.isEmpty(data.FaceRecords)) { + return + } + + // delete images with no or multiple faces + if (data.FaceRecords.length !== 1) { + return this.rekognition.deleteFaces(AWS_REKOGNITION_COLLECTION, _.map(data.FaceRecords, ({ Face }) => Face.FaceId)) + .then(() => this.s3.deleteObject(image)) + } + + indexedFaces++ + }) .catch((error) => Logger.warn(error)) + }, { concurrency: 3 }) + } + + return Promise.all([ deleteFaces.bind(this)(), indexFaces.bind(this)() ]) + .then(() => { + const stop = _.now() + const duration = _.round((stop - start) / 1000) + + return Logger.debug(`Synced reference face collection: ${Faces.length - deletedFaces + indexedFaces} faces available (time = ${duration}s, deleted = ${deletedFaces}, indexed = ${indexedFaces})`) }) - ]) - .then(() => Logger.debug(`Synced reference face collection: ${currentFaces.length - facesToDelete.length + facesToIndex.length} faces available (deleted ${facesToDelete.length}, indexed ${facesToIndex.length})`)) }) }) } checkPhotosOut (photos) { - return Promise.mapSeries(photos, ({ url }) => { - let faceSimilarity = 0 - - return this.rekognition.searchFacesByImage(AWS_REKOGNITION_COLLECTION, url) + return Promise.map(photos, (photo) => { + return savePhoto.bind(this)(photo) + .then((image) => this.rekognition.searchFacesByImage(AWS_REKOGNITION_COLLECTION, image)) .then(({ FaceMatches }) => { - return Promise.mapSeries(FaceMatches, ({ Similarity }) => { - if (!faceSimilarity) { - faceSimilarity = Similarity + photo.similarity = _.round(_.max(_.map(FaceMatches, 'Similarity')), 2) || 0 - return - } - - faceSimilarity = (faceSimilarity + Similarity) / 2 - }) + return photo.similarity }) - .then(() => faceSimilarity) .catch(() => { - return undefined + photo.similarity = 0 + + return photo.similarity }) - }) + }, { concurrency: 3 }) + .then((faceSimilarities) => { + const faceSimilarityMax = _.max(faceSimilarities) + const faceSimilarityMin = _.min(faceSimilarities) + const faceSimilarityMean = _.round(_.mean(_.without(faceSimilarities, 0, undefined)), 2) || 0 + + const like = !_.isEmpty(faceSimilarities) && faceSimilarityMean > 80 + + return { faceSimilarities, faceSimilarityMax, faceSimilarityMin, faceSimilarityMean, like } + }) } - mentalSnapshot (urls) { - return Promise.mapSeries(urls, (url) => { - return request.getAsync(url) - .then(({ body }) => this.s3.putObject(`train/${url.split('/')[ 4 ]}`, body)) + mentalSnapshot (photos) { + return Promise.mapSeries(photos, (photo) => { + const url = parse(photo.url) + if (!url) { + return + } + + const pathname = url.pathname + + const srcKey = pathname + const dstKey = srcKey.replace(`/${AWS_S3_BUCKET}/photos`, 'train') + + return this.s3.copyObject(srcKey, dstKey) }) .then(() => this.syncS3BucketAndRekognitionCollection()) } diff --git a/src/utils/rekognition.js b/src/utils/rekognition.js index 7fe69e8e5..46eabbb31 100644 --- a/src/utils/rekognition.js +++ b/src/utils/rekognition.js @@ -10,24 +10,22 @@ const Promise = require('bluebird') const AWS = require('aws-sdk') -const request = Promise.promisifyAll(require('request').defaults({ encoding: null })) - const defaultOptions = {} class Rekognition { constructor (options = {}) { - this.options = _.defaults(options, defaultOptions) + this._options = _.defaults(options, defaultOptions) - const { region, accessKeyId, secretAccessKey } = this.options + const { region, accessKeyId, secretAccessKey } = this._options AWS.config.update({ region, accessKeyId, secretAccessKey }) - this.rekognition = Promise.promisifyAll(new AWS.Rekognition()) + this._rekognition = Promise.promisifyAll(new AWS.Rekognition()) } listCollections () { const params = {} - return this.rekognition.listCollectionsAsync(params) + return this._rekognition.listCollectionsAsync(params) .then((data) => data.CollectionIds) } @@ -38,7 +36,7 @@ class Rekognition { const params = { CollectionId: collectionId } - return this.rekognition.createCollectionAsync(params) + return this._rekognition.createCollectionAsync(params) } indexFaces (collectionId, bucket, key) { @@ -48,7 +46,7 @@ class Rekognition { const params = { CollectionId: collectionId, - ExternalImageId: key.split('/')[ key.split('/').length - 1 ], + ExternalImageId: Buffer.from(key).toString('base64'), Image: { S3Object: { Bucket: bucket, @@ -57,7 +55,7 @@ class Rekognition { } } - return this.rekognition.indexFacesAsync(params) + return this._rekognition.indexFacesAsync(params) } listFaces (collectionId) { @@ -67,7 +65,13 @@ class Rekognition { const params = { CollectionId: collectionId } - return this.rekognition.listFacesAsync(params) + return this._rekognition.listFacesAsync(params) + .then((data) => { + return Promise.mapSeries(data.Faces, (face) => { + face.ExternalImageId = Buffer.from(face.ExternalImageId, 'base64').toString('ascii') + }) + .then(() => data) + }) } deleteFaces (collectionId, faceIds) { @@ -84,51 +88,42 @@ class Rekognition { FaceIds: faceIds } - return this.rekognition.deleteFacesAsync(params) + return this._rekognition.deleteFacesAsync(params) } - detectFaces (imageUrl) { - if (!imageUrl) { + detectFaces (image) { + if (!image) { return Promise.reject(new Error('invalid arguments')) } - return request.getAsync(imageUrl) - .then(({ body }) => { - const params = { Image: { Bytes: body } } + const params = { Image: { Bytes: image } } - return this.rekognition.detectFacesAsync(params) - }) + return this._rekognition.detectFacesAsync(params) } - detectLabels (imageUrl) { - if (!imageUrl) { + detectLabels (image) { + if (!image) { return Promise.reject(new Error('invalid arguments')) } - return request.getAsync(imageUrl) - .then(({ body }) => { - const params = { Image: { Bytes: body } } + const params = { Image: { Bytes: image } } - return this.rekognition.detectLabelsAsync(params) - }) + return this._rekognition.detectLabelsAsync(params) } - searchFacesByImage (collectionId, imageUrl) { - if (!collectionId || !imageUrl) { + searchFacesByImage (collectionId, image) { + if (!collectionId || !image) { return Promise.reject(new Error('invalid arguments')) } - return request.getAsync(imageUrl) - .then(({ body }) => { - const params = { - CollectionId: collectionId, - FaceMatchThreshold: 10, - MaxFaces: 5, - Image: { Bytes: body } - } + const params = { + CollectionId: collectionId, + FaceMatchThreshold: 10, + MaxFaces: 5, + Image: { Bytes: image } + } - return this.rekognition.searchFacesByImageAsync(params) - }) + return this._rekognition.searchFacesByImageAsync(params) } } diff --git a/src/utils/s3.js b/src/utils/s3.js index cac5b3db3..1a91956a3 100644 --- a/src/utils/s3.js +++ b/src/utils/s3.js @@ -7,19 +7,42 @@ const _ = require('lodash') const Promise = require('bluebird') +const retry = require('bluebird-retry') +const Brakes = require('brakes') + +const Health = require('health-checkup') const AWS = require('aws-sdk') -const defaultOptions = {} +const defaultOptions = { + retry: { max_tries: 2, interval: 1000, throw_original: true }, + breaker: { timeout: 3000, threshold: 80, circuitDuration: 30000 } +} class S3 { constructor (options = {}) { - this.options = _.defaults(options, defaultOptions) + this._options = _.defaults(options, defaultOptions) - const { region, accessKeyId, secretAccessKey } = this.options + const { region, accessKeyId, secretAccessKey } = this._options AWS.config.update({ region, accessKeyId, secretAccessKey }) - this.s3 = Promise.promisifyAll(new AWS.S3()) + this._s3 = Promise.promisifyAll(new AWS.S3()) + + this._breaker = new Brakes(this._options.breaker) + + this._s3.putObjectCircuitBreaker = this._breaker.slaveCircuit((params) => retry(() => this._s3.putObjectAsync(params), this._options.retry)) + this._s3.copyObjectCircuitBreaker = this._breaker.slaveCircuit((params) => retry(() => this._s3.copyObjectAsync(params), this._options.retry)) + this._s3.getObjectCircuitBreaker = this._breaker.slaveCircuit((params) => retry(() => this._s3.getObjectAsync(params), this._options.retry)) + this._s3.deleteObjectCircuitBreaker = this._breaker.slaveCircuit((params) => retry(() => this._s3.deleteObjectAsync(params), this._options.retry)) + this._s3.listObjectsCircuitBreaker = this._breaker.slaveCircuit((params) => retry(() => this._s3.listObjectsAsync(params), this._options.retry)) + + Health.addCheck('s3', () => new Promise((resolve, reject) => { + if (this._breaker.isOpen()) { + return reject(new Error(`circuit breaker is open`)) + } else { + return resolve() + } + })) } putObject (key, data) { @@ -27,20 +50,39 @@ class S3 { return Promise.reject(new Error('invalid arguments')) } - const params = { Bucket: this.options.bucket, Key: key, Body: data } + const params = { Bucket: this._options.bucket, Key: key, Body: data } + + return this._s3.putObjectCircuitBreaker.exec(params) + } + + copyObject (srcKey, dstKey) { + if (!srcKey || !dstKey) { + return Promise.reject(new Error('invalid arguments')) + } + + const params = { Bucket: this._options.bucket, CopySource: srcKey, Key: dstKey } - return this.s3.putObjectAsync(params) + return this._s3.copyObjectCircuitBreaker.exec(params) } getObject (key) { if (!key) { return Promise.reject(new Error('invalid arguments')) + } + const params = { Bucket: this._options.bucket, Key: key } + + return this._s3.getObjectCircuitBreaker.exec(params) + } + + deleteObject (key) { + if (!key) { + return Promise.reject(new Error('invalid arguments')) } - const params = { Bucket: this.options.bucket, Key: key } + const params = { Bucket: this._options.bucket, Key: key } - return this.s3.getObjectAsync(params) + return this._s3.deleteObjectCircuitBreaker.exec(params) } listObjects (prefix, maxKeys = 1000) { @@ -48,17 +90,10 @@ class S3 { return Promise.reject(new Error('invalid arguments')) } - return this.s3.listObjectsAsync({ - Bucket: this.options.bucket, - Prefix: prefix, - MaxKeys: maxKeys - }) - .then((data) => { - return _(data.Contents) - .filter(({ Key }) => Key.split(`${prefix}/`).length > 1 && Key.split(`${prefix}/`)[ 1 ] !== '') - .map(({ Key }) => Key.split(`${prefix}/`)[ 1 ]) - .value() - }) + const params = { Bucket: this._options.bucket, Prefix: prefix, MaxKeys: maxKeys } + + return this._s3.listObjectsCircuitBreaker.exec(params) + .then((data) => _.map(data.Contents, ({ Key }) => Key)) } } diff --git a/test/utils/s3.js b/test/utils/s3.js new file mode 100644 index 000000000..b75c82a85 --- /dev/null +++ b/test/utils/s3.js @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2017, Hugo Freire . + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +describe('S3', () => { + let subject + let AWS + + before(() => { + AWS = td.object([ 'config' ]) + AWS.config.update = td.function() + AWS.S3 = td.constructor([ 'putObject', 'copyObject', 'getObject', 'deleteObject', 'listObjects' ]) + }) + + afterEach(() => td.reset()) + + describe('when constructing', () => { + const region = 'my-region' + const accessKeyId = 'my-access-key-id' + const secretAccessKey = 'my-secret-access-key' + const options = { region, accessKeyId, secretAccessKey } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + }) + + it('should configure AWS access', () => { + const captor = td.matchers.captor() + + td.verify(AWS.config.update(captor.capture()), { times: 1 }) + + const params = captor.value + params.region.should.be.equal(region) + params.accessKeyId.should.be.equal(accessKeyId) + params.secretAccessKey.should.be.equal(secretAccessKey) + }) + }) + + describe('when putting an object', () => { + const bucket = 'my-bucket' + const key = 'my-key' + const data = 'my-data' + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + td.when(AWS.S3.prototype.putObject(), { ignoreExtraArgs: true }).thenCallback() + + subject.putObject(key, data) + }) + + it('should call AWS S3 putObject', () => { + td.verify(AWS.S3.prototype.putObject({ + Bucket: bucket, + Key: key, + Body: data + }), { ignoreExtraArgs: true, times: 1 }) + }) + }) + + describe('when putting an object with invalid arguments', () => { + const bucket = 'my-bucket' + const key = undefined + const data = undefined + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + td.when(AWS.S3.prototype.putObject(), { ignoreExtraArgs: true }).thenCallback() + }) + + it('should reject with invalid arguments error', () => { + return subject.putObject(key, data) + .catch((error) => { + error.should.be.instanceOf(Error) + error.message.should.be.equal('invalid arguments') + }) + }) + }) + + describe('when copying an object', () => { + const bucket = 'my-bucket' + const srcKey = 'my-source-key' + const dstKey = 'my-destination-key' + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + td.when(AWS.S3.prototype.copyObject(), { ignoreExtraArgs: true }).thenCallback() + + subject.copyObject(srcKey, dstKey) + }) + + it('should call AWS S3 copyObject', () => { + td.verify(AWS.S3.prototype.copyObject({ + Bucket: bucket, + CopySource: srcKey, + Key: dstKey + }), { ignoreExtraArgs: true, times: 1 }) + }) + }) + + describe('when copying an object with invalid arguments', () => { + const bucket = 'my-bucket' + const srcKey = undefined + const dstKey = undefined + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + td.when(AWS.S3.prototype.copyObject(), { ignoreExtraArgs: true }).thenCallback() + }) + + it('should reject with invalid arguments error', () => { + return subject.copyObject(srcKey, dstKey) + .catch((error) => { + error.should.be.instanceOf(Error) + error.message.should.be.equal('invalid arguments') + }) + }) + }) + + describe('when getting an object', () => { + const bucket = 'my-bucket' + const key = 'my-key' + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + subject.getObject(key) + }) + + it('should call AWS S3 getObject', () => { + td.verify(AWS.S3.prototype.getObject({ Bucket: bucket, Key: key }), { + ignoreExtraArgs: true, + times: 1 + }) + }) + }) + + describe('when getting an object with invalid arguments', () => { + const bucket = 'my-bucket' + const key = undefined + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + td.when(AWS.S3.prototype.getObject(), { ignoreExtraArgs: true }).thenCallback() + }) + + it('should reject with invalid arguments error', () => { + return subject.getObject(key) + .catch((error) => { + error.should.be.instanceOf(Error) + error.message.should.be.equal('invalid arguments') + }) + }) + }) + + describe('when deleting an object', () => { + const bucket = 'my-bucket' + const key = 'my-key' + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + subject.deleteObject(key) + }) + + it('should call AWS S3 deleteObject', () => { + td.verify(AWS.S3.prototype.deleteObject({ Bucket: bucket, Key: key }), { + ignoreExtraArgs: true, + times: 1 + }) + }) + }) + + describe('when deleting an object with invalid arguments', () => { + const bucket = 'my-bucket' + const key = undefined + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + td.when(AWS.S3.prototype.deleteObject(), { ignoreExtraArgs: true }).thenCallback() + }) + + it('should reject with invalid arguments error', () => { + return subject.deleteObject(key) + .catch((error) => { + error.should.be.instanceOf(Error) + error.message.should.be.equal('invalid arguments') + }) + }) + }) + + describe('when listing objects', () => { + const bucket = 'my-bucket' + const prefix = 'my-prefix' + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + subject.listObjects(prefix) + }) + + it('should call AWS S3 listObjectsAsync', () => { + td.verify(AWS.S3.prototype.listObjects({ + Bucket: bucket, + Prefix: prefix, + MaxKeys: 1000 + }), { ignoreExtraArgs: true, times: 1 }) + }) + }) + + describe('when listing objects with invalid arguments', () => { + const bucket = 'my-bucket' + const prefix = undefined + const options = { bucket } + + beforeEach(() => { + td.replace('aws-sdk', AWS) + + const S3 = require('../../src/utils/s3') + subject = new S3(options) + + td.when(AWS.S3.prototype.listObjects(), { ignoreExtraArgs: true }).thenCallback() + }) + + it('should reject with invalid arguments error', () => { + return subject.listObjects(prefix) + .catch((error) => { + error.should.be.instanceOf(Error) + error.message.should.be.equal('invalid arguments') + }) + }) + }) +})