diff --git a/bin/storj-cron.js b/bin/storj-cron.js new file mode 100755 index 000000000..9c881dc3f --- /dev/null +++ b/bin/storj-cron.js @@ -0,0 +1,27 @@ +#!/usr/bin/env node + +'use strict'; + +const async = require('async'); +const program = require('commander'); +const Config = require('../lib/config'); +const StorageEventsCron = require('../lib/cron/storage-events'); + +program.version(require('../package').version); +program.option('-c, --config ', 'path to the config file'); +program.option('-d, --datadir ', 'path to the data directory'); +program.parse(process.argv); + +var config = new Config(process.env.NODE_ENV || 'develop', program.config, program.datadir); + +var jobs = [ + new StorageEventsCron(config) +]; + +async.eachSeries(jobs, function(job, next) { + job.start(next); +}, function(err) { + if (err) { + throw err; + } +}); diff --git a/lib/config.js b/lib/config.js index 74d0276f6..40d340591 100644 --- a/lib/config.js +++ b/lib/config.js @@ -57,6 +57,7 @@ const DEFAULTS = { from: 'robot@storj.io' }, application: { + unknownReportThreshold: 0.3, // one third unknown reports activateSIP6: false, powOpts: { retargetPeriod: 10000, // milliseconds diff --git a/lib/cron/storage-events.js b/lib/cron/storage-events.js new file mode 100644 index 000000000..489b5c9f1 --- /dev/null +++ b/lib/cron/storage-events.js @@ -0,0 +1,223 @@ +'use strict'; + +const assert = require('assert'); +const CronJob = require('cron').CronJob; +const Config = require('../config'); +const Storage = require('storj-service-storage-models'); +const log = require('../logger'); + +function StorageEventsCron(config) { + if (!(this instanceof StorageEventsCron)) { + return new StorageEventsCron(config); + } + + assert(config instanceof Config, 'Invalid config supplied'); + + this._config = config; +} + +StorageEventsCron.CRON_TIME = '* */10 * * * *'; // every ten minutes +StorageEventsCron.MAX_RUN_TIME = 600000; // 10 minutes +StorageEventsCron.FINALITY_TIME = 10800000; // 3 hours + +StorageEventsCron.prototype.start = function(callback) { + log.info('starting the storage events cron'); + + this.storage = new Storage( + this._config.storage.mongoUrl, + this._config.storage.mongoOpts, + { logger: log } + ); + + this.job = new CronJob({ + cronTime: StorageEventsCron.CRON_TIME, + onTick: this.run.bind(this), + start: false, + timeZone: 'UTC' + }); + + this.job.start(); + + setImmediate(callback); +}; + +StorageEventsCron.prototype._resolveCodes = function(event, user) { + const threshold = this._config.application.unknownReportThreshold; + + let success = event.success; + let successModified = false; + let unknown = success ? false : true; + if (unknown) { + resolveCodes(); + } + + function resolveCodes() { + const failureCode = 1100; + + const clientCode = event.clientReport ? + event.clientReport.exchangeResultCode : undefined; + const farmerCode = event.farmerReport ? + event.farmerReport.exchangeResultCode : undefined; + + /* jshint eqeqeq: false */ + if (farmerCode == failureCode && !clientCode) { + success = false; + unknown = false; + return; + } + + if (farmerCode == failureCode && clientCode == failureCode) { + success = false; + unknown = false; + return; + } + + if (!farmerCode && clientCode == failureCode) { + success = false; + unknown = false; + return; + } + + if (user.exceedsUnknownReportsThreshold(threshold)) { + successModified = true; + success = true; + unknown = true; + return; + } + } + + return { + success: success, + successModified: successModified, + unknown: unknown + }; + +}; + +StorageEventsCron.prototype._resolveEvent = function(event, callback) { + + this.storage.models.User.findOne({_id: event.user}, (err, user) => { + if (err) { + return callback(err); + } + + const {success, successModified, unknown} = this._resolveCodes(event); + + if (successModified) { + // TODO also give reputation points to farmer for successful + // transfer for the storage event + event.success = success; + event.save((err) => { + if (err) { + return callback(err); + } + finalize(); + }); + } else { + finalize(); + } + + function finalize() { + user.updateUnknownReports(unknown, event.timestamp, (err) => { + if (err) { + return callback(err); + } + + callback(null, event.timestamp); + }); + } + }); +}; + +StorageEventsCron.prototype._run = function(lastTimestamp, callback) { + + const StorageEvent = this.storage.models.StorageEvent; + const finalityTime = Date.now() - StorageEventsCron.FINALITY_TIME; + + const cursor = StorageEvent.find({ + timestamp: { + $lt: finalityTime, + $gt: lastTimestamp + }, + user: { + $exists: true + } + }).sort({timestamp: 1}).cursor(); + + const timeout = setTimeout(() => { + finish(new Error('Job exceeded max duration')); + }, StorageEventsCron.MAX_RUN_TIME); + + let callbackCalled = false; + + function finish(err) { + clearTimeout(timeout); + cursor.close(); + if (!callbackCalled) { + callbackCalled = true; + callback(err, lastTimestamp); + } + } + + cursor.on('error', finish); + + cursor.on('data', (event) => { + cursor.pause(); + this._resolveEvent(event, (err, _lastTimestamp) => { + if (err) { + return finish(err); + } + lastTimestamp = _lastTimestamp; + cursor.resume(); + }); + }); + + cursor.on('end', finish); +}; + +StorageEventsCron.prototype.run = function() { + const name = 'StorageEventsFinalityCron'; + const Cron = this.storage.models.CronJob; + Cron.lock(name, StorageEventsCron.MAX_RUN_TIME, (err, locked, res) => { + if (err) { + return log.error('%s lock failed, reason: %s', name, err.message); + } + if (!locked) { + return log.warn('%s already running', name); + } + + log.info('Starting %s cron job', name); + + let lastTimestamp = new Date(0); + if (res && + res.value && + res.value.rawData && + res.value.rawData.lastTimestamp) { + lastTimestamp = new Date(res.value.rawData.lastTimestamp); + } else { + log.warn('%s cron has unknown lastTimestamp', name); + } + + this._run(lastTimestamp, (err, _lastTimestamp) => { + if (err) { + let message = err.message ? err.message : 'unknown'; + log.error('Error running %s, reason: %s', name, message); + } + + log.info('Stopping %s cron', name); + const rawData = {}; + if (_lastTimestamp) { + rawData.lastTimestamp = _lastTimestamp.getTime(); + } + + Cron.unlock(name, rawData, (err) => { + if (err) { + return log.error('%s unlock failed, reason: %s', name, err.message); + } + }); + + }); + }); +}; + +module.exports = StorageEventsCron; diff --git a/lib/monitor/index.js b/lib/monitor/index.js index 9df2c266b..28cd21027 100644 --- a/lib/monitor/index.js +++ b/lib/monitor/index.js @@ -5,7 +5,8 @@ const assert = require('assert'); const crypto = require('crypto'); const storj = require('storj-lib'); const MonitorConfig = require('./config'); -const Storage = require('storj-service-storage-models'); +const StorageModels = require('storj-service-storage-models'); +const points = StorageModels.constants.POINTS; const ComplexClient = require('storj-complex').createClient; const MongoDBStorageAdapter = require('storj-mongodb-adapter'); const ms = require('ms'); @@ -44,7 +45,7 @@ Monitor.MAX_SIGINT_WAIT = 5000; Monitor.prototype.start = function(callback) { log.info('Farmer monitor service is starting'); - this.storage = new Storage( + this.storage = new StorageModels( this._config.storage.mongoUrl, this._config.storage.mongoOpts, { logger: log } @@ -152,6 +153,33 @@ Monitor.prototype._saveShard = function(shard, destination, callback) { }); }; +Monitor.prototype._createStorageEvent = function(token, + shardHash, + shardBytes, + client, + farmer) { + const StorageEvent = this.storage.models.StorageEvent; + + const storageEvent = new StorageEvent({ + token: token, + user: null, + client: client, + farmer: farmer, // farmer storing the new mirror + timestamp: Date.now(), + downloadBandwidth: 0, + storage: shardBytes, + shardHash: shardHash, + success: false + }); + + storageEvent.save((err) => { + if (err) { + log.warn('_createStorageEvent: Error saving event, ' + + 'reason: %s', err.message); + } + }); +}; + Monitor.prototype._transferShard = function(shard, state, callback) { const source = state.sources[0]; const destination = state.destinations[0]; @@ -187,6 +215,11 @@ Monitor.prototype._transferShard = function(shard, state, callback) { const farmer = storj.Contact(destination.contact); + const token = pointer.token; + const shardHash = shard.hash; + const shardBytes = contract.get('data_size'); + this._createStorageEvent(token, shardHash, shardBytes, source.nodeID, farmer.nodeID); + this.network.getMirrorNodes([pointer], [farmer], (err) => { if (err) { log.warn('Unable to mirror to farmer %s, reason: %s', @@ -247,6 +280,36 @@ Monitor.prototype._replicateFarmer = function(contact) { }); }; +Monitor.prototype._giveOfflinePenalty = function(contact) { + contact.recordPoints(points.OFFLINE); + contact.save((err) => { + if (err) { + this._logger.error('_giveOfflinePenalty: Unable to save contact %s ' + + 'to update reputation, reason: %s', + contact._id, err.message); + } + }); +}; + +Monitor.prototype._markStorageEventsEnded = function(contact) { + const StorageEvent = this.storage.models.StorageEvent; + StorageEvent.update({ + farmer: contact._id, + farmerEnd: { $exists: false } + }, { + $currentDate: { + farmerEnd: true + } + }, { + multi: true + }, (err) => { + if (err) { + log.error('Unable to update farmer %s storage events with farmerEnd, ' + + 'reason: %s', contact._id, err.message); + } + }); +}; + Monitor.prototype.run = function() { if (this._running) { return this.wait(); @@ -317,7 +380,10 @@ Monitor.prototype.run = function() { if (contactData.timeoutRate >= timeoutRateThreshold) { log.warn('Shards need replication, farmer: %s, timeoutRate: %s', contact.nodeID, contactData.timeoutRate); + this._replicateFarmer(contact); + this._giveOfflinePenalty(contactData); + this._markStorageEventsEnded(contactData); } } else { diff --git a/lib/server/middleware/farmer-auth.js b/lib/server/middleware/farmer-auth.js index d94cb1f0e..89018c8d0 100644 --- a/lib/server/middleware/farmer-auth.js +++ b/lib/server/middleware/farmer-auth.js @@ -99,6 +99,8 @@ function authFarmer(req, res, next) { return next(new errors.BadRequestError('Invalid signature header')); } + req.farmerNodeID = nodeID; + next(); } diff --git a/lib/server/routes/buckets.js b/lib/server/routes/buckets.js index 14d20d883..816336e97 100644 --- a/lib/server/routes/buckets.js +++ b/lib/server/routes/buckets.js @@ -182,6 +182,159 @@ BucketsRouter.prototype.createBucket = function(req, res, next) { }); }; +BucketsRouter.prototype._getShardHashesByBucketId = function(userId, bucketId, callback) { + const bucketObjectId = new mongoose.Types.ObjectId(bucketId); + + this.storage.models.Bucket.aggregate([ + { + $match: { + user: userId, + _id: bucketObjectId + } + }, + { + $project: { + _id: 1 + } + }, + { + $lookup: { + from: 'bucketentries', + localField: '_id', + foreignField: 'bucket', + as: 'bucketentries' + } + }, + { + $unwind: { + path: '$bucketentries' + } + }, + { + $project: { + _id: 1, + bucketEntryId: '$bucketentries._id', + frame: '$bucketentries.frame' + } + }, + { + $lookup: { + from: 'frames', + localField: 'frame', + foreignField: '_id', + as: 'frames' + } + }, + { + $unwind: { + path: '$frames' + } + }, + { + $project: { + _id: 1, + shardPointers: '$frames.shards' + } + }, + { + $unwind: { + path: '$shardPointers', + } + }, + { + $lookup: { + from: 'pointers', + localField: 'shardPointers', + foreignField: '_id', + as: 'pointers' + } + }, + { + $project: { + _id: 0, + shardHash: '$pointers.hash' + } + }, + { + $unwind: { + path: '$shardHash' + } + } + ], (err, result) => { + if (err) { + return callback(err); + } + callback(null, result.map((a) => a.shardHash)); + }); +}; + +BucketsRouter.prototype._getShardHashesByBucketEntryId = function(userId, bucketEntryId, callback) { + const bucketEntryObjectId = new mongoose.Types.ObjectId(bucketEntryId); + + this.storage.models.BucketEntry.aggregate([ + { + $match: { + _id: bucketEntryObjectId + } + }, + { + $project: { + _id: 1, + frame: 1 + } + }, + { + $lookup: { + from: 'frames', + localField: 'frame', + foreignField: '_id', + as: 'frames' + } + }, + { + $unwind: { + path: '$frames' + } + }, + { + $project: { + _id: 1, + shardPointers: '$frames.shards' + } + }, + { + $unwind: { + path: '$shardPointers', + } + }, + { + $lookup: { + from: 'pointers', + localField: 'shardPointers', + foreignField: '_id', + as: 'pointers' + } + }, + { + $project: { + _id: 0, + shardHash: '$pointers.hash' + } + }, + { + $unwind: { + path: '$shardHash' + } + } + ], (err, result) => { + if (err) { + return callback(err); + } + callback(null, result.map((a) => a.shardHash)); + }); +}; + + /** * Destroys the user's bucket by ID * @param {http.IncomingMessage} req @@ -193,81 +346,72 @@ BucketsRouter.prototype.destroyBucketById = function(req, res, next) { const BucketEntry = this.storage.models.BucketEntry; const StorageEvent = this.storage.models.StorageEvent; - const bucketObjectId = new mongoose.Types.ObjectId(req.params.id); - analytics.track(req.headers.dnt, { userId: req.user.uuid, event: 'Bucket Destroyed' }); - BucketEntry.aggregate([ - { - $match: { - bucket: bucketObjectId - } - }, - { - $lookup: { - from: 'frames', - localField: 'frame', - foreignField: '_id', - as: 'frameData' - } - }, - { - $unwind: { - path: '$frameData' - } - }, - { - $project: { - _id: 0, - bucket: '$bucket', - bucketEntry: '$_id', - user: {$literal: req.user._id}, - downloadBandwidth: {$literal: 0}, - storage: {$multiply: [-1, '$frameData.size']}, - timestamp: {$add: [new Date(), 0]} - } - }], - function(err, storageEvents) { + const userId = req.user._id; + const bucketId = req.params.id; + + this._getShardHashesByBucketId(userId, bucketId, (err, hashes) => { + if (err) { + log.warn('destroyBucketById: storage event aggregation failed, reason: %s', + err.message); + return next(new errors.InternalError(err.message)); + } + + if (!hashes || !hashes.length) { + return next(new errors.NotFoundError('Bucket not found')); + } + + Bucket.findOne({ + _id: bucketId, + user: userId + }, (err, bucket) => { if (err) { - log.warn('destroyBucketById: storage event aggregation failed, reason: %s', - err.message); + return next(new errors.InternalError(err.message)); + } + + if (!bucket) { + return next(new errors.NotFoundError('Bucket not found')); } - Bucket.findOne({ - _id: req.params.id, - user: req.user._id - }, function(err, bucket) { + // Because we don't have atomic transactions, we'll first update + // the storage events as ended. If this job does not complete, this + // action can be safely repeated to complete the task. + StorageEvent.update({ + user: userId, + shardHash: { $in: hashes }, + userDeleted: { $exists: false } + }, { + $currentDate: { + userDeleted: true + } + }, { + multi: true + }, (err) => { if (err) { return next(new errors.InternalError(err.message)); } - if (!bucket) { - return next(new errors.NotFoundError('Bucket not found')); - } - - bucket.remove(function(err) { + BucketEntry.remove({ bucket: req.params.id }, (err) => { if (err) { return next(new errors.InternalError(err.message)); } - BucketEntry.remove({ bucket: req.params.id }, (err) => { + + // As the final step, remove the bucket itself. + bucket.remove((err) => { if (err) { - log.error('Unable to remove bucket entries, reason: %s', - err.message); + return next(new errors.InternalError(err.message)); } - StorageEvent.collection.insert(storageEvents, function(err) { - if (err) { - log.warn('Unable to save storage events, reason: %s', - err.message); - } + + res.status(204).end(); }); }); - }); - res.status(204).end(); }); }); + }); }; /** @@ -453,7 +597,6 @@ BucketsRouter.prototype.createEntryFromFrame = function(req, res, next) { const Frame = this.storage.models.Frame; const Bucket = this.storage.models.Bucket; const BucketEntry = this.storage.models.BucketEntry; - const StorageEvent = this.storage.models.StorageEvent; if (req.body.filename && req.body.filename.length > constants.MAX_BUCKETENTRYNAME) { @@ -511,23 +654,6 @@ BucketsRouter.prototype.createEntryFromFrame = function(req, res, next) { return next(new errors.InternalError(err.message)); } - var fileCreationStorageEventData = { - bucket: req.params.id, - bucketEntry: entry._id, - user: req.user._id, - downloadBandwidth: 0, - storage: frame.storageSize - }; - - var fileCreationStorageEvent = new StorageEvent(fileCreationStorageEventData); - - fileCreationStorageEvent.save(function(err) { - if (err) { - log.warn('createEntryFromFrame: failed to save storage event, reason: %s', - err.message); - } - }); - res.send(merge(entry.toObject(), { size: frame.size })); }); }); @@ -1043,7 +1169,6 @@ BucketsRouter.prototype._requestRetrievalPointer = function(item, meta, done) { BucketsRouter.prototype._getPointersFromEntry = function(entry, opts, user, done) { const self = this; - const StorageEvent = this.storage.models.StorageEvent; const Pointer = this.storage.models.Pointer; let pQuery = { @@ -1079,23 +1204,6 @@ BucketsRouter.prototype._getPointersFromEntry = function(entry, opts, } }); - var fileDownloadStorageEventData = { - bucket: entry.bucket, - bucketEntry: entry._id, - user: user._id, - downloadBandwidth: bytes, - storage: 0 - }; - - var fileDownloadStorageEvent = new StorageEvent(fileDownloadStorageEventData); - - fileDownloadStorageEvent.save(function(err) { - if (err) { - log.warn('createEntryFromFrame: failed to save storage event, reason: %s', - err.message); - } - }); - async.mapLimit(pointers, 6, function(sPointer, next) { self._getRetrievalToken(sPointer, { excludeFarmers: opts.excludeFarmers @@ -1110,6 +1218,34 @@ BucketsRouter.prototype._getPointersFromEntry = function(entry, opts, }); }; +BucketsRouter.prototype._createStorageEvents = function(user, results, callback) { + const StorageEvent = this.storage.models.StorageEvent; + const cb = callback ? callback : function(err) { + if (err) { + log.warn('_createStorageEvents: failed to save storage event, reason: %s', + err.message); + } + }; + async.eachSeries(results, (item, next) => { + if (!item.token) { + // skip events for missing shards + return next(); + } + const storageEvent = new StorageEvent({ + token: item.token, + user: user._id, + client: user._id, + farmer: item.farmer.nodeID, + timestamp: Date.now(), + shardHash: item.hash, + downloadBandwidth: item.size, + storage: 0, + success: false + }); + storageEvent.save(next); + }, cb); +}; + /** * @callback BucketsRouter~_getPointersFromEntryCallback * @param {Error|null} [error] @@ -1204,6 +1340,9 @@ BucketsRouter.prototype.getFile = function(req, res, next) { return next(err); } + // Create storage events for each pointer/shard + self._createStorageEvents(user, result); + res.send(result); }); }); @@ -1271,53 +1410,61 @@ BucketsRouter.prototype.removeFile = function(req, res, next) { const BucketEntry = this.storage.models.BucketEntry; const StorageEvent = this.storage.models.StorageEvent; + const userId = req.user._id; + Bucket.findOne({ - _id: req.params.id, - user: req.user._id - }, function(err, bucket) { - if (err) { - return next(new errors.InternalError(err.message)); - } - - if (!bucket) { - return next(new errors.NotFoundError('Bucket not found')); - } - - BucketEntry.findOne({ - bucket: bucket._id, - _id: req.params.file - }).populate('frame').exec(function(err, entry) { - if (err) { - return next(err); - } - - if (!entry) { - return next(new errors.NotFoundError('File not found')); - } - - entry.remove(function(err) { - if (err) { - return next(new errors.InternalError(err.message)); - } - - var fileRemovalEventData = { - bucket: req.params.id, - bucketEntry: req.params.file, - user: req.user._id, - downloadBandwidth: 0, - storage: -1 * entry.frame.size - }; + _id: req.params.id, + user: userId + }, (err, bucket) => { + if (err) { + return next(new errors.InternalError(err.message)); + } + + if (!bucket) { + return next(new errors.NotFoundError('Bucket not found')); + } + + BucketEntry.findOne({ + bucket: bucket._id, + _id: req.params.file + }, (err, entry) => { + if (err) { + return next(err); + } + + if (!entry) { + return next(new errors.NotFoundError('File not found')); + } - var fileRemovalStorageEvent = new StorageEvent(fileRemovalEventData); + this._getShardHashesByBucketEntryId(userId, entry._id, (err, hashes) => { + if (err) { + return next(new errors.InternalError(err.message)); + } - fileRemovalStorageEvent.save(function(err) { - if (err) { - log.warn('createEntryFromFrame: failed to save storage event, reason: %s', - err.message); - } - }); + StorageEvent.update({ + user: userId, + shardHash: { $in: hashes }, + userDeleted: { $exists: false } + }, { + $currentDate: { + userDeleted: true + } + }, { + multi: true + }, (err) => { + if (err) { + return next(new errors.InternalError(err.message)); + } + + entry.remove(function(err) { + if (err) { + return next(new errors.InternalError(err.message)); + } - res.status(204).end(); + res.status(204).end(); + }); + + }); }); }); }); diff --git a/lib/server/routes/contacts.js b/lib/server/routes/contacts.js index b2e2e3fa4..9071755ce 100644 --- a/lib/server/routes/contacts.js +++ b/lib/server/routes/contacts.js @@ -83,7 +83,7 @@ ContactsRouter.prototype.createContact = function(req, res, next) { ContactsRouter.prototype.patchContactByNodeID = function(req, res, next) { const Contact = this.storage.models.Contact; - const nodeID = req.headers['x-node-id']; + const nodeID = req.farmerNodeID; const data = {}; if (req.body.address) { diff --git a/lib/server/routes/frames.js b/lib/server/routes/frames.js index 7ff41d8b8..7d5d76077 100644 --- a/lib/server/routes/frames.js +++ b/lib/server/routes/frames.js @@ -214,6 +214,28 @@ FramesRouter.prototype._publishContract = function(nodes, contract, audit, callb }); }; +FramesRouter.prototype._createStorageEvent = function(token, user, farmer, pointer) { + const StorageEvent = this.storage.models.StorageEvent; + const storageEvent = new StorageEvent({ + token: token, + user: user._id, + client: user._id, + farmer: farmer.nodeID, + downloadBandwidth: 0, + storage: pointer.size, + timestamp: Date.now(), + shardHash: pointer.hash, + success: false, + }); + + storageEvent.save(function(err) { + if (err) { + log.warn('createEntryFromFrame: failed to save storage event, reason: %s', + err.message); + } + }); +}; + /** * Negotiates a storage contract and adds the shard to the frame * @param {http.IncomingMessage} req @@ -448,6 +470,10 @@ FramesRouter.prototype.addShardToFrame = function(req, res, next) { if (err) { return done(new errors.InternalError(err.message)); } + + // Keep track of this storage event + self._createStorageEvent(token, req.user, farmer, pointer); + res.send({ hash: req.body.hash, token: token, diff --git a/lib/server/routes/reports.js b/lib/server/routes/reports.js index 005c95a94..399052df2 100644 --- a/lib/server/routes/reports.js +++ b/lib/server/routes/reports.js @@ -3,7 +3,11 @@ const Router = require('./index'); const log = require('../../logger'); const middleware = require('storj-service-middleware'); +const farmerMiddleware = require('../middleware/farmer-auth'); +const rawBody = require('../middleware/raw-body'); const errors = require('storj-service-error-types'); +const StorageModels = require('storj-service-storage-models'); +const points = StorageModels.constants.POINTS; const inherits = require('util').inherits; const BucketsRouter = require('./buckets'); const constants = require('../../constants'); @@ -24,10 +28,84 @@ function ReportsRouter(options) { Router.apply(this, arguments); this.getLimiter = middleware.rateLimiter(options.redis); + this.userAuthMiddlewares = middleware.authenticate(this.storage); + this.farmerRawBodyMiddleware = rawBody; } inherits(ReportsRouter, Router); +ReportsRouter.prototype.authMiddleware = function(req, res, next) { + /* jshint sub:true */ + const isUser = (req.headers['authorization'] !== undefined || + req.headers['x-pubkey'] !== undefined); + const isFarmer = (req.headers['x-node-id'] !== undefined); + + if (isUser) { + this.userAuthMiddlewares[0](req, res, (err) => { + if (err) { + return next(err); + } + this.userAuthMiddlewares[1](req, res, next); + }); + } else if (isFarmer) { + this.farmerRawBodyMiddleware(req, res, (err) => { + if (err) { + return next(err); + } + farmerMiddleware.authFarmer(req, res, next); + }); + } else { + next(new errors.NotAuthorizedError('No authentication strategy detected')); + } +}; + +ReportsRouter.prototype.updateReputation = function(nodeID, points) { + this.storage.models.Contact.findOne({_id: nodeID}, (err, contact) => { + if (err || !contact) { + log.warn('updateReputation: Error trying to find contact ' + + ' %s, reason: %s', nodeID, err ? err.message : 'unknown'); + return; + } + contact.recordPoints(points).save((err) => { + if (err) { + log.warn('updateReputation: Error saving contact ' + + ' %s, reason: %s', nodeID, err.message); + } + }); + }); +}; + +ReportsRouter.prototype.validateExchangeReport = function(report) { + if (!Number.isFinite(report.exchangeStart)) { + return false; + } + if (!Number.isFinite(report.exchangeEnd)) { + return false; + } + + if (report.exchangeResultCode !== 1100 && + report.exchangeResultCode !== 1000) { + return false; + } + + // TODO: Check timestamp is reasonable + + const validMessages = [ + 'FAILED_INTEGRITY', + 'SHARD_DOWNLOADED', + 'SHARD_UPLOADED', + 'DOWNLOAD_ERROR', + 'TRANSFER_FAILED', + 'MIRROR_SUCCESS' + ]; + + if (!validMessages.includes(report.exchangeResultMessage)) { + return false; + } + + return true; +}; + /** * Creates an exchange report * @param {http.IncomingMessage} req @@ -35,54 +113,100 @@ inherits(ReportsRouter, Router); * @param {Function} next */ ReportsRouter.prototype.createExchangeReport = function(req, res, next) { - const self = this; - var exchangeReport = new this.storage.models.ExchangeReport(req.body); - var projection = { - hash: true, - contracts: true - }; - - this.storage.models.Shard.find({ - hash: exchangeReport.dataHash - }, projection, function(err, shards) { + const token = req.body.token; + + /* jshint maxstatements: 30, maxcomplexity: 15 */ + this.storage.models.StorageEvent.findOne({token: token}, (err, event) => { if (err) { return next(new errors.InternalError(err.message)); } - if (!shards || !shards.length) { - return next(new errors.NotFoundError('Shard not found for report')); + if (!event) { + return next(new errors.NotFoundError('Storage event not found')); } - // TODO: Add signature/identity verification + // TODO: Check that event isn't closed for reports - // NB: Kick off mirroring if needed - self._handleExchangeReport(exchangeReport, (err) => { - /* istanbul ignore next */ - if (err) { - return log.warn(err.message); - } - }); - exchangeReport.save(function(err) { - if (err) { - return next(new errors.BadRequestError(err.message)); + const report = { + exchangeStart: req.body.exchangeStart, + exchangeEnd: req.body.exchangeEnd, + exchangeResultCode: req.body.exchangeResultCode, + exchangeResultMessage: req.body.exchangeResultMessage + }; + + if (!this.validateExchangeReport(report)) { + return next(new errors.BadRequestError('Invalid exchange report')); + } + + const isClientReport = req.user ? + (event.client === req.user.id) : req.farmerNodeID ? + (event.client === req.farmerNodeID) : false; + const isFarmerReport = req.farmerNodeID ? + (event.farmer === req.farmerNodeID) : false; + + if (!isClientReport && !isFarmerReport) { + return next(new errors.NotAuthorizedError('Not authorized to report')); + } + + let modified = false; + + const hasClientReport = event.clientReport ? + !!event.clientReport.exchangeResultCode : false; + const hasFarmerReport = event.farmerReport ? + !!event.farmerReport.exchangeResultCode : false; + + if (isClientReport && !hasClientReport) { + modified = true; + event.clientReport = report; + if (Number(req.body.exchangeResultCode) === 1000) { + event.success = true; + this.updateReputation(event.farmer, points.TRANSFER_SUCCESS); + } else { + this.updateReputation(event.farmer, points.TRANSFER_FAILURE); } + } else if (isFarmerReport && !hasFarmerReport) { + modified = true; + event.farmerReport = report; + } + + if (modified) { + event.save((err) => { + if (err) { + return next(new errors.InternalError(err.message)); + } + + // Kick off mirroring if needed + this._handleExchangeReport({ + dataHash: req.body.dataHash, + exchangeResultMessage: req.body.exchangeResultMessage + }, event, (err) => { + /* istanbul ignore next */ + if (err) { + return log.warn(err.message); + } + }); + + res.status(201).send({}); + }); + + } else { + res.status(200).send({}); + } - res.status(201).send({}); - }); }); }; /** * @private */ -ReportsRouter.prototype._handleExchangeReport = function(report, callback) { +ReportsRouter.prototype._handleExchangeReport = function(report, prevEvent, callback) { const {dataHash, exchangeResultMessage} = report; switch (exchangeResultMessage) { case 'MIRROR_SUCCESS': case 'SHARD_UPLOADED': case 'DOWNLOAD_ERROR': - this._triggerMirrorEstablish(constants.M_REPLICATE, dataHash, callback); + this._triggerMirrorEstablish(constants.M_REPLICATE, dataHash, prevEvent, callback); break; default: callback(new Error('Exchange result type will not trigger action')); @@ -101,11 +225,41 @@ ReportsRouter._sortByResponseTime = function(a, b) { return (aTime === bTime) ? 0 : (aTime > bTime) ? 1 : -1; }; +ReportsRouter.prototype._createStorageEvent = function(token, + clientNodeID, + farmerNodeID, + storage, + shardHash) { + const StorageEvent = this.storage.models.StorageEvent; + + const storageEvent = new StorageEvent({ + token: token, + user: null, + client: clientNodeID, // farmer that already has the mirror + farmer: farmerNodeID, // farmer storing the new mirror + timestamp: Date.now(), + downloadBandwidth: 0, + storage: storage, + shardHash: shardHash, + success: false + }); + + storageEvent.save((err) => { + if (err) { + log.warn('_createStorageEvent: Error saving event, ' + + 'reason: %s', err.message); + } + }); +}; + /** * Loads some mirrors for the hash and establishes them * @private */ -ReportsRouter.prototype._triggerMirrorEstablish = function(n, hash, done) { +ReportsRouter.prototype._triggerMirrorEstablish = function(n, + hash, + prevEvent, + done) { const self = this; let item = null; @@ -190,6 +344,14 @@ ReportsRouter.prototype._triggerMirrorEstablish = function(n, hash, done) { return callback(new Error('Failed to get pointer')); } + let farmer = pointer.farmer; + let token = pointer.token; + self._createStorageEvent(token, + farmer.nodeID, + mirror.contact, + prevEvent.storage, + hash); + callback(null, pointer, mirror, contact); }); } @@ -230,7 +392,7 @@ ReportsRouter.prototype.getContactById = BucketsRouter.prototype.getContactById; */ ReportsRouter.prototype._definitions = function() { return [ - ['POST', '/reports/exchanges', this.getLimiter(limiter(1000)), middleware.rawbody, + ['POST', '/reports/exchanges', this.getLimiter(limiter(1000)), this.authMiddleware.bind(this), this.createExchangeReport.bind(this)] ]; }; diff --git a/package.json b/package.json index f5e553598..1074557f8 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "commander": "^2.9.0", "concat-stream": "^1.6.0", "cors": "^2.7.1", + "cron": "^1.3.0", "csv-write-stream": "^2.0.0", "elliptic": "^6.0.2", "express": "^4.13.3", @@ -87,7 +88,7 @@ "storj-service-error-types": "^1.2.0", "storj-service-mailer": "^1.0.0", "storj-service-middleware": "^1.3.1", - "storj-service-storage-models": "^10.2.2", + "storj-service-storage-models": "^10.3.0", "through": "^2.3.8" } } diff --git a/test/cron/storage-events.unit.js b/test/cron/storage-events.unit.js new file mode 100644 index 000000000..9f0d2cf78 --- /dev/null +++ b/test/cron/storage-events.unit.js @@ -0,0 +1,562 @@ +'use strict'; + +const sinon = require('sinon'); +const expect = require('chai').expect; +const log = require('../../lib/logger'); +const proxyquire = require('proxyquire'); +const StorageEventsCron = require('../../lib/cron/storage-events'); +const EventEmitter = require('events').EventEmitter; +const Config = require('../../lib/config'); + +describe('StorageEventsCron', function() { + describe('@constructor', function() { + it('will initialize', function() { + var config = new Config('__tmptest'); + const cron = new StorageEventsCron(config); + expect(cron._config); + }); + }); + + describe('#start', function() { + it('will initialize storage, job, and start job', function(done) { + function Storage(mongoUrl, mongoOpts, options) { + expect(options).to.eql({logger: log}); + const url = 'mongodb://127.0.0.1:27017/__storj-bridge-test'; + expect(mongoUrl).to.eql(url); + expect(mongoOpts).to.eql({}); + } + const TestStorageEventsCron = proxyquire('../../lib/cron/storage-events', { + 'storj-service-storage-models': Storage + }); + var config = new Config('__tmptest'); + const cron = new TestStorageEventsCron(config); + cron.start(() => { + done(); + }); + }); + }); + + describe('#_resolveCodes', function() { + var config = new Config('__tmptest'); + it('failed: farmer(false), client(unknown)', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: {}, + farmerReport: { + exchangeResultCode: 1100 + } + }; + const user = {}; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(false); + expect(unknown).to.equal(false); + expect(successModified).to.equal(false); + }); + + it('failed: farmer(false), client(false)', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: { + exchangeResultCode: 1100 + }, + farmerReport: { + exchangeResultCode: 1100 + } + }; + const user = {}; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(false); + expect(unknown).to.equal(false); + expect(successModified).to.equal(false); + }); + + it('failed: farmer(unknown), client(false)', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: { + exchangeResultCode: 1100 + }, + farmerReport: {} + }; + const user = {}; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(false); + expect(unknown).to.equal(false); + expect(successModified).to.equal(false); + }); + + it('success: farmer(true), client(false), > threshold', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: { + exchangeResultCode: 1100 + }, + farmerReport: { + exchangeResultCode: 1000 + } + }; + const user = { + exceedsUnknownReportsThreshold: sinon.stub().returns(true) + }; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(true); + expect(unknown).to.equal(true); + expect(successModified).to.equal(true); + }); + + it('success: farmer(unknown), client(unknown), > threshold', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: {}, + farmerReport: {} + }; + const user = { + exceedsUnknownReportsThreshold: sinon.stub().returns(true) + }; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(true); + expect(unknown).to.equal(true); + expect(successModified).to.equal(true); + }); + + it('success: farmer(true), client(false), > threshold', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: { + exchangeResultCode: 1100 + }, + farmerReport: { + exchangeResultCode: 1000 + } + }; + const user = { + exceedsUnknownReportsThreshold: sinon.stub().returns(true) + }; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(true); + expect(unknown).to.equal(true); + expect(successModified).to.equal(true); + }); + + it('failed: farmer(true), client(false), < threshold', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: { + exchangeResultCode: 1100 + }, + farmerReport: { + exchangeResultCode: 1000 + } + }; + const user = { + exceedsUnknownReportsThreshold: sinon.stub().returns(false) + }; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(false); + expect(unknown).to.equal(true); + expect(successModified).to.equal(false); + }); + + it('failed: farmer(unknown), client(unknown), < threshold', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: {}, + farmerReport: {} + }; + const user = { + exceedsUnknownReportsThreshold: sinon.stub().returns(false) + }; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(false); + expect(unknown).to.equal(true); + expect(successModified).to.equal(false); + }); + + it('failed: farmer(true), client(unknown), < threshold', function() { + const cron = new StorageEventsCron(config); + const event = { + success: false, + clientReport: {}, + farmerReport: { + exchangeResultCode: 1000 + } + }; + const user = { + exceedsUnknownReportsThreshold: sinon.stub().returns(false) + }; + const {success, successModified, unknown} = cron._resolveCodes(event, user); + expect(success).to.equal(false); + expect(unknown).to.equal(true); + expect(successModified).to.equal(false); + }); + + }); + + describe('#_resolveEvent', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + var config = new Config('__tmptest'); + it('handle error from user find query', function(done) { + const cron = new StorageEventsCron(config); + cron.storage = { + models: { + User: { + findOne: sandbox.stub().callsArgWith(1, new Error('test')) + } + } + }; + const event = { + user: 'user@domain.tld', + }; + cron._resolveEvent(event, (err, timestamp) => { + expect(err.message).to.equal('test'); + expect(timestamp).to.equal(undefined); + done(); + }); + }); + it('save update event status', function(done) { + const cron = new StorageEventsCron(config); + const user = { + updateUnknownReports: sandbox.stub().callsArgWith(2) + }; + cron.storage = { + models: { + User: { + findOne: sandbox.stub().callsArgWith(1, null, user) + } + } + }; + cron._resolveCodes = sandbox.stub().returns({ + success: true, + successModified: true, + unknown: false + }); + const now = new Date(); + const event = { + user: 'user@domain.tld', + timestamp: now, + success: false, + save: sandbox.stub().callsArgWith(0) + }; + cron._resolveEvent(event, (err, timestamp) => { + if (err) { + return done(err); + } + expect(event.save.callCount).to.equal(1); + expect(event.success).to.equal(true); + expect(timestamp).to.equal(now); + done(); + }); + }); + it('will handle error from saving event status', function(done) { + const cron = new StorageEventsCron(config); + const user = { + updateUnknownReports: sandbox.stub().callsArgWith(2) + }; + cron.storage = { + models: { + User: { + findOne: sandbox.stub().callsArgWith(1, null, user) + } + } + }; + cron._resolveCodes = sandbox.stub().returns({ + success: true, + successModified: true, + unknown: false + }); + const now = new Date(); + const event = { + user: 'user@domain.tld', + timestamp: now, + success: false, + save: sandbox.stub().callsArgWith(0, new Error('test')) + }; + cron._resolveEvent(event, (err, timestamp) => { + expect(err.message).to.equal('test'); + expect(timestamp).to.equal(undefined); + done(); + }); + }); + it.skip('will give reputation points', function() { + }); + it('will update user unknown report rates', function(done) { + const cron = new StorageEventsCron(config); + const user = { + updateUnknownReports: sandbox.stub().callsArgWith(2) + }; + cron.storage = { + models: { + User: { + findOne: sandbox.stub().callsArgWith(1, null, user) + } + } + }; + const unknown = false; + cron._resolveCodes = sandbox.stub().returns({ + success: true, + successModified: true, + unknown: unknown + }); + const now = new Date(); + const event = { + user: 'user@domain.tld', + timestamp: now, + success: false, + save: sandbox.stub().callsArgWith(0) + }; + cron._resolveEvent(event, (err, timestamp) => { + expect(user.updateUnknownReports.callCount).to.equal(1); + expect(user.updateUnknownReports.args[0][0]).to.equal(unknown); + expect(user.updateUnknownReports.args[0][1]).to.equal(now); + expect(timestamp).to.equal(now); + done(); + }); + }); + it('will handle error from user unknown report update', function(done) { + const cron = new StorageEventsCron(config); + const user = { + updateUnknownReports: sandbox.stub().callsArgWith(2, new Error('test')) + }; + cron.storage = { + models: { + User: { + findOne: sandbox.stub().callsArgWith(1, null, user) + } + } + }; + const unknown = false; + cron._resolveCodes = sandbox.stub().returns({ + success: true, + successModified: true, + unknown: unknown + }); + const now = new Date(); + const event = { + user: 'user@domain.tld', + timestamp: now, + success: false, + save: sandbox.stub().callsArgWith(0) + }; + cron._resolveEvent(event, (err, timestamp) => { + expect(err.message).to.equal('test'); + expect(timestamp).to.equal(undefined); + done(); + }); + }); + }); + + describe('#_run', function() { + var config = new Config('__tmptest'); + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('it will close cursor and stop on timeout', function(done) { + const clock = sandbox.useFakeTimers(); + const cron = new StorageEventsCron(config); + const cursor = new EventEmitter(); + cursor.close = sandbox.stub(); + cron.storage = { + models: { + StorageEvent: { + find: sandbox.stub().returns({ + sort: sandbox.stub().returns({ + cursor: sandbox.stub().returns(cursor) + }) + }) + } + } + }; + const timestamp = new Date(); + cron._run(timestamp, (err) => { + expect(err.message).to.equal('Job exceeded max duration'); + done(); + }); + clock.tick(StorageEventsCron.MAX_RUN_TIME + 1); + }); + + it('it will resolve each event', function(done) { + const cron = new StorageEventsCron(config); + cron._resolveEvent = sandbox.stub(); + const cursor = new EventEmitter(); + cursor.close = sandbox.stub(); + cursor.pause = sandbox.stub(); + cron.storage = { + models: { + StorageEvent: { + find: sandbox.stub().returns({ + sort: sandbox.stub().returns({ + cursor: sandbox.stub().returns(cursor) + }) + }) + } + } + }; + const timestamp = new Date(); + const events = [ + { + 'token': 'f571c06a871857b3fb38187875609ec5718acd0b', + }, + { + 'token': '688a4efc515f17967ac809f963e54e1024beab41' + }, + { + 'token': 'd2f399c8fa5039b10b35e73d826e43d34c00453f' + } + ]; + cron._run(timestamp, (err) => { + if (err) { + return done(err); + } + expect(cron._resolveEvent.callCount).to.equal(3); + expect(cron._resolveEvent.args[0][0]).to.equal(events[0]); + expect(cron._resolveEvent.args[1][0]).to.equal(events[1]); + expect(cron._resolveEvent.args[2][0]).to.equal(events[2]); + done(); + }); + events.forEach((event) => { + cursor.emit('data', event); + }); + cursor.emit('end'); + }); + }); + + describe('#run', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + var config = new Config('__tmptest'); + + it('will handle error from lock', function() { + sandbox.stub(log, 'error'); + sandbox.stub(log, 'warn'); + sandbox.stub(log, 'info'); + const cron = new StorageEventsCron(config); + cron._run = sandbox.stub(); + cron.storage = { + models: { + CronJob: { + lock: sandbox.stub().callsArgWith(2, new Error('test')) + } + } + }; + cron.run(); + expect(log.error.callCount).to.equal(1); + expect(log.error.args[0][0]).to.match(/lock failed/); + expect(cron._run.callCount).to.equal(0); + }); + + it('will bail if lock can not be acquired', function() { + sandbox.stub(log, 'error'); + sandbox.stub(log, 'warn'); + sandbox.stub(log, 'info'); + const cron = new StorageEventsCron(config); + cron._run = sandbox.stub(); + cron.storage = { + models: { + CronJob: { + lock: sandbox.stub().callsArgWith(2, null, false) + } + } + }; + cron.run(); + expect(log.warn.callCount).to.equal(1); + expect(log.warn.args[0][0]).to.match(/already running/); + expect(cron._run.callCount).to.equal(0); + }); + + it('will log error from _run and unlock ', function() { + sandbox.stub(log, 'error'); + sandbox.stub(log, 'warn'); + sandbox.stub(log, 'info'); + const cron = new StorageEventsCron(config); + cron._run = sandbox.stub().callsArgWith(1, new Error('test')); + cron.storage = { + models: { + CronJob: { + lock: sandbox.stub().callsArgWith(2, null, true), + unlock: sandbox.stub().callsArgWith(2, null) + } + } + }; + cron.run(); + expect(log.error.callCount).to.equal(1); + expect(log.error.args[0][0]).to.match(/Error running/); + expect(cron.storage.models.CronJob.lock.callCount).to.equal(1); + expect(cron.storage.models.CronJob.unlock.callCount).to.equal(1); + expect(cron._run.callCount).to.equal(1); + }); + + it('will call _run with lastTimestamp from lock and unlock', function() { + sandbox.stub(log, 'error'); + sandbox.stub(log, 'warn'); + sandbox.stub(log, 'info'); + const now = new Date(); + const then = new Date(now.getTime() + 100000); + const cron = new StorageEventsCron(config); + cron._run = sandbox.stub().callsArgWith(1, null, then); + const res = { + value: { + rawData: { + lastTimestamp: now.getTime() + } + } + }; + const unlock = sandbox.stub().callsArgWith(2, null); + cron.storage = { + models: { + CronJob: { + lock: sandbox.stub().callsArgWith(2, null, true, res), + unlock: unlock + } + } + }; + cron.run(); + expect(cron._run.callCount).to.equal(1); + expect(cron._run.args[0][0]).to.be.instanceOf(Date); + expect(cron._run.args[0][0]).to.eql(now); + expect(unlock.callCount).to.equal(1); + expect(unlock.args[0][0]).to.equal('StorageEventsFinalityCron'); + expect(unlock.args[0][1]).to.eql({ + lastTimestamp: then.getTime() + }); + }); + + it('will log error from unlock', function() { + sandbox.stub(log, 'error'); + sandbox.stub(log, 'warn'); + sandbox.stub(log, 'info'); + const timestamp = new Date().toISOString(); + const cron = new StorageEventsCron(config); + cron._run = sandbox.stub().callsArgWith(1, new Error('test')); + const res = { + value: { + rawData: { + lastTimestamp: timestamp + } + } + }; + const unlock = sandbox.stub().callsArgWith(2, new Error('test')); + cron.storage = { + models: { + CronJob: { + lock: sandbox.stub().callsArgWith(2, null, true, res), + unlock: unlock + } + } + }; + cron.run(); + expect(unlock.callCount).to.equal(1); + expect(unlock.args[0][0]).to.equal('StorageEventsFinalityCron'); + }); + }); + +}); diff --git a/test/monitor/index.unit.js b/test/monitor/index.unit.js index 6c78f593f..b1cdb4d4e 100644 --- a/test/monitor/index.unit.js +++ b/test/monitor/index.unit.js @@ -187,6 +187,50 @@ describe('Monitor', function() { }); }); + describe('#_giveOfflinePenalty', function() { + it('it will record offline points', function() { + const monitor = new Monitor(config); + const contact = { + recordPoints: sandbox.stub(), + save: sandbox.stub() + }; + monitor._giveOfflinePenalty(contact); + expect(contact.recordPoints.callCount).to.equal(1); + expect(contact.recordPoints.args[0][0]).to.equal(-1000); + expect(contact.save.callCount).to.equal(1); + }); + }); + + describe('#_markStorageEventsEnded', function() { + it('it will update events as ended', function() { + const monitor = new Monitor(config); + const update = sandbox.stub().callsArgWith(3, null); + monitor.storage = { + models: { + StorageEvent: { + update: update + } + } + }; + const contact = { + _id: 'e211c3b8a710f50bd26644de5128b194be3904e1', + }; + monitor._markStorageEventsEnded(contact); + expect(update.callCount).to.equal(1); + expect(update.args[0][0]).to.eql({ + farmer: 'e211c3b8a710f50bd26644de5128b194be3904e1', + farmerEnd: { + $exists: false + } + }); + expect(update.args[0][1]).to.eql({ + $currentDate: { + farmerEnd: true + } + }); + }); + }); + describe('#_fetchSources', function() { it('it will query and sort contacts', function(done) { sandbox.stub(log, 'error'); @@ -348,6 +392,45 @@ describe('Monitor', function() { }); }); + describe('#_createStorageEvent', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('it will create event and save', function() { + function StorageEvent(options) { + expect(options).to.eql({ + token: '527b83da713c30e56b3b0dd74a14152f24475909', + user: null, + client: 'cec5a2d61f4e82f6c73f44388b5798c64784d68f', + farmer: 'b0789385f486a0d0d07f32b0a96b313202cf4031', + timestamp: 1509141238388, + downloadBandwidth: 0, + storage: 1024, + shardHash: '88c99bf39dcc693fe1a2a232601a37fec8d466b3', + success: false + }); + } + const save = sinon.stub(); + StorageEvent.prototype.save = save; + const monitor = new Monitor(config); + monitor.storage = { + models: { + StorageEvent: StorageEvent + } + }; + let token = '527b83da713c30e56b3b0dd74a14152f24475909'; + let shardHash = '88c99bf39dcc693fe1a2a232601a37fec8d466b3'; + let shardBytes = 1024; + let source = 'cec5a2d61f4e82f6c73f44388b5798c64784d68f'; + let destination = 'b0789385f486a0d0d07f32b0a96b313202cf4031'; + const clock = sandbox.useFakeTimers(); + clock.tick(1509141238388); + monitor._createStorageEvent(token, shardHash, + shardBytes, source, destination); + expect(save.callCount).to.equal(1); + }); + }); + describe('#_transferShard', function() { const sandbox = sinon.sandbox.create(); afterEach(() => sandbox.restore()); @@ -435,6 +518,7 @@ describe('Monitor', function() { getRetrievalPointer: sinon.stub().callsArgWith(2, null, pointer), getMirrorNodes: sinon.stub().callsArgWith(2, new Error('timeout')) }; + monitor._createStorageEvent = sinon.stub(); const contract = new storj.Contract(); const shard = { getContract: sandbox.stub().returns(contract) @@ -464,6 +548,7 @@ describe('Monitor', function() { .to.equal(pointer); expect(monitor.network.getMirrorNodes.args[0][1][0]) .to.be.instanceOf(storj.Contact); + expect(monitor._createStorageEvent.callCount).to.equal(1); expect(log.warn.callCount).to.equal(1); expect(monitor._transferShard.callCount).to.equal(2); expect(monitor._saveShard.callCount).to.equal(0); @@ -477,22 +562,30 @@ describe('Monitor', function() { sandbox.stub(log, 'warn'); const monitor = new Monitor(config); monitor._saveShard = sinon.stub().callsArg(2); - const pointer = {}; + const pointer = { + token: 'token' + }; monitor.network = { getRetrievalPointer: sinon.stub().callsArgWith(2, null, pointer), getMirrorNodes: sinon.stub().callsArgWith(2, null, {}) }; - const contract = new storj.Contract(); + monitor._createStorageEvent = sinon.stub(); + const contract = new storj.Contract({ + data_size: 1337 + }); const shard = { - getContract: sandbox.stub().returns(contract) + getContract: sandbox.stub().returns(contract), + hash: 'hash' }; const contact = storj.Contact({ + nodeID: '27d71722a9843831b22964ebcf42e6bc5b8624de', address: '127.0.0.1', port: 100000 }); const mirror = { contract: {}, contact: { + nodeID: '0c764f44017688a5ee54af195939260331341114', address: '128.0.0.1', port: 100000 } @@ -509,14 +602,20 @@ describe('Monitor', function() { expect(log.error.callCount).to.equal(0); expect(log.warn.callCount).to.equal(0); expect(monitor._transferShard.callCount).to.equal(1); + expect(monitor._createStorageEvent.callCount).to.equal(1); + expect(monitor._createStorageEvent.args[0][0]).to.equal('token'); + expect(monitor._createStorageEvent.args[0][1]).to.equal('hash'); + expect(monitor._createStorageEvent.args[0][2]).to.equal(1337); + expect(monitor._createStorageEvent.args[0][3]) + .to.equal(state.sources[0].nodeID); + expect(monitor._createStorageEvent.args[0][4]) + .to.equal(state.destinations[0].contact.nodeID); expect(monitor._saveShard.callCount).to.equal(1); expect(monitor._saveShard.args[0][0]).to.equal(shard); expect(monitor._saveShard.args[0][1]).to.equal(state.destinations[0]); done(); }); - }); - }); describe('#_replicateShard', function() { @@ -717,14 +816,12 @@ describe('Monitor', function() { expect(monitor.wait.callCount).to.equal(1); }); - it('will log error when querying contacts', function() { const monitor = new Monitor(config); sandbox.stub(log, 'error'); sandbox.stub(log, 'info'); - const exec = sandbox.stub().callsArgWith(0, new Error('Mongo error')); const sort = sandbox.stub().returns({ exec: exec @@ -844,6 +941,8 @@ describe('Monitor', function() { }; monitor.wait = sandbox.stub(); monitor._replicateFarmer = sinon.stub(); + monitor._giveOfflinePenalty = sinon.stub(); + monitor._markStorageEventsEnded = sinon.stub(); monitor.run(); expect(find.callCount).to.equal(1); @@ -891,6 +990,12 @@ describe('Monitor', function() { expect(monitor._replicateFarmer.args[0][0]) .to.be.instanceOf(storj.Contact); + expect(monitor._giveOfflinePenalty.callCount).to.equal(1); + expect(monitor._giveOfflinePenalty.args[0][0].recordPoints); + + expect(monitor._markStorageEventsEnded.callCount).to.equal(1); + expect(monitor._markStorageEventsEnded.args[0][0]._id); + expect(monitor.network.ping.callCount).to.equal(3); expect(monitor.network.ping.args[0][0]).to.be.instanceOf(storj.Contact); }); diff --git a/test/server/routes/buckets.unit.js b/test/server/routes/buckets.unit.js index 0f756cf0d..c44846741 100644 --- a/test/server/routes/buckets.unit.js +++ b/test/server/routes/buckets.unit.js @@ -521,17 +521,14 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketEntryAggregate = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, null, [{}]); - var _bucketFindOne = sinon.stub( + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, new Error('Failed to lookup bucket')); bucketsRouter.destroyBucketById(request, response, function(err) { - _bucketEntryAggregate.restore(); - _bucketFindOne.restore(); expect(err.message).to.equal('Failed to lookup bucket'); done(); }); @@ -547,23 +544,20 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketEntryAggregate = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, null, [{}]); - var _bucketFindOne = sinon.stub( + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, null); bucketsRouter.destroyBucketById(request, response, function(err) { - _bucketEntryAggregate.restore(); - _bucketFindOne.restore(); expect(err.message).to.equal('Bucket not found'); done(); }); }); - it('should log error if unable to remove bucket entries', function() { + it('should give error if unable to remove bucket entries', function(done) { sandbox.stub(log, 'error'); var request = httpMocks.createRequest({ method: 'DELETE', @@ -574,10 +568,9 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - sandbox.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, null, [{}]); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' @@ -588,11 +581,13 @@ describe('BucketsRouter', function() { bucketsRouter.storage.models.BucketEntry, 'remove' ).callsArgWith(1, new Error('test')); - bucketsRouter.destroyBucketById(request, response); - expect(log.error.callCount).to.equal(1); + bucketsRouter.destroyBucketById(request, response, function(err) { + expect(err).to.be.instanceOf(Error); + done(); + }); }); - it('should internal error if deletion fails', function(done) { + it('should internal error if storage event delete fails', function(done) { var request = httpMocks.createRequest({ method: 'DELETE', url: '/buckets/:bucket_id' @@ -605,23 +600,20 @@ describe('BucketsRouter', function() { var bucket = new bucketsRouter.storage.models.Bucket({ user: someUser._id }); - var _bucketEntryAggregate = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, null, [{}]); - var _bucketFindOne = sinon.stub( + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, bucket); - var _bucketRemove = sinon.stub(bucket, 'remove').callsArgWith( - 0, - new Error('Failed to remove bucket') + sandbox.stub(bucketsRouter.storage.models.StorageEvent, 'update').callsArgWith( + 3, + new Error('test') ); bucketsRouter.destroyBucketById(request, response, function(err) { - _bucketEntryAggregate.restore(); - _bucketFindOne.restore(); - _bucketRemove.restore(); - expect(err.message).to.equal('Failed to remove bucket'); + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); done(); }); }); @@ -639,26 +631,24 @@ describe('BucketsRouter', function() { var bucket = new bucketsRouter.storage.models.Bucket({ user: someUser._id }); - var _bucketEntryAggregate = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, null, [{}]); - var _bucketFindOne = sinon.stub( - bucketsRouter.storage.models.Bucket, - 'findOne' - ).callsArgWith(1, null, bucket); - var _bucketRemove = sinon.stub(bucket, 'remove').callsArg(0); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); + sandbox.stub(bucketsRouter.storage.models.Bucket, 'findOne') + .callsArgWith(1, null, bucket); + sandbox.stub(bucketsRouter.storage.models.StorageEvent, 'update') + .callsArgWith(3, null); + sandbox.stub(bucketsRouter.storage.models.BucketEntry, 'remove') + .callsArgWith(1, null); + sandbox.stub(bucket, 'remove').callsArg(0); response.on('end', function() { - _bucketEntryAggregate.restore(); - _bucketFindOne.restore(); - _bucketRemove.restore(); expect(response.statusCode).to.equal(204); done(); }); bucketsRouter.destroyBucketById(request, response); }); - it('should log warning if aggregation fails', function(done) { + it('should give error if remove fails', function(done) { sandbox.stub(log, 'warn'); var request = httpMocks.createRequest({ method: 'DELETE', @@ -669,29 +659,28 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketEntryAggregate = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, new Error('Storage event aggregation failed'), [{}, {}]); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); var bucket = new bucketsRouter.storage.models.Bucket({ user: someUser._id }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, bucket); - var _bucketRemove = sinon.stub(bucket, 'remove').callsArg(0); - response.on('end', function() { - _bucketEntryAggregate.restore(); - _bucketFindOne.restore(); - _bucketRemove.restore(); - expect(log.warn.callCount).to.equal(1); + sandbox.stub( + bucketsRouter.storage.models.BucketEntry, + 'remove' + ).callsArgWith(1, new Error('test')); + bucketsRouter.destroyBucketById(request, response, function(err) { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); done(); }); - bucketsRouter.destroyBucketById(request, response); }); - it('should log warning if storage events fail to save', function(done) { + it('should give error if bucket remove fails', function(done) { sandbox.stub(log, 'warn'); var request = httpMocks.createRequest({ method: 'DELETE', @@ -702,36 +691,26 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketEntryAggregate = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, null, [{}, {}]); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); var bucket = new bucketsRouter.storage.models.Bucket({ user: someUser._id }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, bucket); - var _bucketRemove = sinon.stub(bucket, 'remove').callsArg(0, null); - var _bucketEntryRemove = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'remove' - ).callsArgWith(1, null); - var _storageEventInsert = sinon.stub( - bucketsRouter.storage.models.StorageEvent.collection, - 'insert' - ).callsArgWith(1, new Error('Storage events failed to save')); + sandbox.stub(bucketsRouter.storage.models.BucketEntry, 'remove') + .callsArgWith(1, null); + sandbox.stub(bucket, 'remove').callsArgWith(0, new Error('test')); response.on('end', function() { - _bucketEntryAggregate.restore(); - _bucketFindOne.restore(); - _bucketRemove.restore(); - _bucketEntryRemove.restore(); - _storageEventInsert.restore(); - expect(log.warn.callCount).to.equal(1); done(); }); - bucketsRouter.destroyBucketById(request, response); + bucketsRouter.destroyBucketById(request, response, function(err) { + expect(err).to.be.instanceOf(Error); + done(); + }); }); it('should respond with 204 when storage events get saved', function(done) { @@ -744,38 +723,31 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketEntryAggregate = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'aggregate' - ).callsArgWith(1, null, [{}, {}]); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketId') + .callsArgWith(2, null, hashes); var bucket = new bucketsRouter.storage.models.Bucket({ user: someUser._id }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, bucket); - var _bucketRemove = sinon.stub(bucket, 'remove').callsArg(0, null); - var _bucketEntryRemove = sinon.stub( - bucketsRouter.storage.models.BucketEntry, - 'remove' + sandbox.stub(bucket, 'remove').callsArg(0, null); + sandbox.stub( + bucketsRouter.storage.models.BucketEntry, + 'remove' ).callsArgWith(1, null); - var _storageEventInsert = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.StorageEvent.collection, 'insert' ).callsArgWith(1, null); response.on('end', function() { - _bucketEntryAggregate.restore(); - _bucketFindOne.restore(); - _bucketRemove.restore(); - _bucketEntryRemove.restore(); - _storageEventInsert.restore(); expect(response.statusCode).to.equal(204); done(); }); bucketsRouter.destroyBucketById(request, response); }); - }); describe('#updateBucketById', function() { @@ -1759,74 +1731,6 @@ describe('BucketsRouter', function() { bucketsRouter.createEntryFromFrame(request, response); }); - it('should return internal error if storage event save fails', function(done) { - sandbox.stub(log,'warn'); - var request = httpMocks.createRequest({ - method: 'POST', - url: '/buckets/:bucket_id/files', - body: { - frame: 'frameid', - mimetype: 'application/octet-stream', - filename: 'somefilename', - hmac: { - value: 'f891be8e91491e4aeeb193e9e3afb49e83b6cc18df2be9732dd62545' + - 'ec5d318076ef86adc5771dc4b7b1ce8802bb3b9dce9f7c5a438afd1b1f52f' + - 'b5e37e3f5c8', - type: 'sha512' - } - }, - params: { - id: 'bucketid' - } - }); - request.user = someUser; - var response = httpMocks.createResponse({ - req: request, - eventEmitter: EventEmitter - }); - sandbox.stub( - bucketsRouter.storage.models.Bucket, - 'findOne' - ).callsArgWith(1, null, { - _id: 'bucketid' - }); - sandbox.stub( - bucketsRouter.storage.models.Frame, - 'findOne' - ).callsArgWith(1, null, { - _id: 'frameid', - locked: false, - lock: sandbox.stub().callsArg(0), - }); - const hmac = { - value: 'f891be8e91491e4aeeb193e9e3afb49e83b6cc18df2be9732dd62545' + - 'ec5d318076ef86adc5771dc4b7b1ce8802bb3b9dce9f7c5a438afd1b1f52f' + - 'b5e37e3f5c8', - type: 'sha512' - }; - var entry = { frame: 'frameid', bucket: 'bucketid', hmac: hmac}; - sandbox.stub( - bucketsRouter.storage.models.BucketEntry, - 'create' - ).callsArgWith(1, null, { - _id: 'bucketentryid', - frame: 'frameid', - toObject: sandbox.stub().returns(entry) - }); - function StorageEvent() {} - StorageEvent.prototype.save = sandbox.stub().callsArgWith(0, new Error('test')); - sandbox.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); - response.on('end', function() { - expect(log.warn.callCount).to.equal(1); - done(); - }); - bucketsRouter.createEntryFromFrame(request, response); - }); - }); describe('#_getBucketById', function() { @@ -1886,9 +1790,11 @@ describe('BucketsRouter', function() { }); describe('#getBucketEntryById', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); it('should internal error if query fails', function(done) { - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' ).returns({ @@ -1898,14 +1804,13 @@ describe('BucketsRouter', function() { exec: sinon.stub().callsArgWith(0, new Error('Query failed')) }); bucketsRouter.getBucketEntryById('bucketid', 'entryid', function(err) { - _bucketFindOne.restore(); expect(err.message).to.equal('Query failed'); done(); }); }); it('should not found error if entry not found', function(done) { - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' ).returns({ @@ -1915,17 +1820,14 @@ describe('BucketsRouter', function() { exec: sinon.stub().callsArgWith(0, null, null) }); bucketsRouter.getBucketEntryById('bucketid', 'entryid', function(err) { - _bucketFindOne.restore(); expect(err.message).to.equal('Entry not found'); done(); }); }); it('should return the bucket entry', function(done) { - var _bucketEntry = new bucketsRouter.storage.models.BucketEntry({ - - }); - var _bucketFindOne = sinon.stub( + var _bucketEntry = new bucketsRouter.storage.models.BucketEntry({}); + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' ).returns({ @@ -1935,7 +1837,6 @@ describe('BucketsRouter', function() { exec: sinon.stub().callsArgWith(0, null, _bucketEntry) }); bucketsRouter.getBucketEntryById('bucketid', 'entryid', function(e, be) { - _bucketFindOne.restore(); expect(be).to.equal(_bucketEntry); done(); }); @@ -1944,30 +1845,30 @@ describe('BucketsRouter', function() { }); describe('#getPointersForEntry', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); it('should internal error if query fails', function(done) { - var _pointerFind = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Pointer, 'find' ).callsArgWith(1, new Error('Query failed')); bucketsRouter.getPointersForEntry({ frame: { shards: [] } }, function(err) { - _pointerFind.restore(); expect(err.message).to.equal('Query failed'); done(); }); }); it('should return pointers', function(done) { - var _pointerFind = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Pointer, 'find' ).callsArgWith(1, null, []); bucketsRouter.getPointersForEntry({ frame: { shards: [] } }, function(err, p) { - _pointerFind.restore(); expect(Array.isArray(p)).to.equal(true); done(); }); @@ -2609,6 +2510,57 @@ describe('BucketsRouter', function() { }); + describe('#_createStorageEvents', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('will create storage event for each download pointer', function(done) { + const testBucketsRouter = new BucketsRouter( + require('../../_fixtures/router-opts') + ); + const clock = sandbox.useFakeTimers(); + clock.tick(1509149787342); + const user = { + _id: 'userid' + }; + const results = [{ + token: 'a359c0108d3c8f6b61cda41c931dc7bf556ef337', + farmer: { + nodeID: 'e7dc097f28fcc845a57b106c7bb98dafd9ac11b7', + }, + hash: '523e34ca44d2ce9a2051578727b4793b1972ccca', + size: 1337 + }]; + function StorageEvent(options) { + expect(options).to.eql({ + token: 'a359c0108d3c8f6b61cda41c931dc7bf556ef337', + user: 'userid', + client: 'userid', + farmer: 'e7dc097f28fcc845a57b106c7bb98dafd9ac11b7', + timestamp: Date.now(), + shardHash: '523e34ca44d2ce9a2051578727b4793b1972ccca', + downloadBandwidth: 1337, + storage: 0, + success: false + }); + } + const save = sandbox.stub().callsArg(0); + StorageEvent.prototype.save = save; + testBucketsRouter.storage = { + models: { + StorageEvent: StorageEvent + } + }; + testBucketsRouter._createStorageEvents(user, results, (err) => { + if (err) { + return done(err); + } + expect(save.callCount).to.equal(1); + done(); + }); + }); + }); + describe('#_getPointersFromEntry', function() { const sandbox = sinon.sandbox.create(); afterEach(() => sandbox.restore()); @@ -2796,104 +2748,6 @@ describe('BucketsRouter', function() { done(); }); }); - - it('should callback with results', function(done) { - sandbox.stub( - bucketsRouter.storage.models.Pointer, - 'find' - ).returns({ - skip: function() { - return this; - }, - limit: function() { - return this; - }, - sort: function() { - return this; - }, - exec: sandbox.stub().callsArgWith(0, null, [{size: 10}]) - }); - function StorageEvent() {} - StorageEvent.prototype.save = sinon.stub().callsArgWith(0, null); - sandbox.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); - var token = {}; - sandbox.stub( - bucketsRouter, - '_getRetrievalToken' - ).callsArgWith(2, null, token); - bucketsRouter._getPointersFromEntry({ - bucket: 'bucketid', - _id: 'bucketentryid', - frame: { shards: [] } - }, { - skip: 6, - limit: 12 - }, someUser, function(err, results) { - if (err) { - return done(err); - } - expect(results[0]).to.equal(token); - expect(bucketsRouter.storage.models.StorageEvent.prototype.save.callCount).to.equal(1); - expect(bucketsRouter.storage.models.StorageEvent.args[0][0]).to.eql({ - bucket: 'bucketid', - bucketEntry: 'bucketentryid', - downloadBandwidth: 10, - storage: 0, - user: 'gordon@storj.io' - }); - done(); - }); - }); - - it('should throw error if storage event save fails', function(done) { - sandbox.stub(log, 'warn'); - const testUser = new bucketsRouter.storage.models.User({ - _id: 'testuser@storj.io', - hashpass: storj.utils.sha256('password') - }); - testUser.isDownloadRateLimited = sandbox.stub().returns(false); - testUser.recordDownloadBytes = sandbox.stub().callsArgWith(1, null); - sandbox.stub( - bucketsRouter.storage.models.Pointer, - 'find' - ).returns({ - skip: function() { - return this; - }, - limit: function() { - return this; - }, - sort: function() { - return this; - }, - exec: sandbox.stub().callsArgWith(0, null, [{size: 10}]) - }); - sandbox.stub( - bucketsRouter, - '_getRetrievalToken' - ).callsArgWith(2, null, {}); - function StorageEvent() {} - StorageEvent.prototype.save = sinon.stub().callsArgWith(0, new Error('test')); - sandbox.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); - bucketsRouter._getPointersFromEntry({ - frame: { shards: [] } - }, { - skip: 6, - limit: 12 - }, testUser, function() { - expect(log.warn.callCount).to.equal(1); - done(); - }); - }); - }); describe('#getFile', function() { @@ -2934,6 +2788,7 @@ describe('BucketsRouter', function() { 'findOne' ).callsArgWith(1, null, testUser); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err).to.be.instanceOf(errors.TransferRateError); expect(err.message) @@ -2957,6 +2812,7 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err.message).to.equal('Not authorized'); done(); @@ -2988,7 +2844,7 @@ describe('BucketsRouter', function() { bucketsRouter.storage.models.User, 'findOne' ).callsArgWith(1, null, someUser); - + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err.message).to.equal('Query failed'); done(); @@ -3021,6 +2877,7 @@ describe('BucketsRouter', function() { 'findOne' ).callsArgWith(1, new Error('user test')); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err).to.be.instanceOf(errors.InternalError); expect(err.message).to.equal('user test'); @@ -3054,6 +2911,7 @@ describe('BucketsRouter', function() { 'findOne' ).callsArgWith(1, null, null); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err).to.be.instanceOf(errors.NotFoundError); expect(err.message).to.equal('User not found for bucket'); @@ -3086,6 +2944,7 @@ describe('BucketsRouter', function() { 'findOne' ).callsArgWith(1, null, someUser); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { _bucketFindOne.restore(); expect(err.message).to.equal('Bucket not found'); @@ -3131,6 +2990,7 @@ describe('BucketsRouter', function() { exec: sandbox.stub().callsArgWith(0, new Error('Query failed')) }); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err.message).to.equal('Query failed'); done(); @@ -3175,6 +3035,7 @@ describe('BucketsRouter', function() { exec: sandbox.stub().callsArgWith(0, null, null) }); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err.message).to.equal('File not found'); done(); @@ -3227,6 +3088,8 @@ describe('BucketsRouter', function() { bucketsRouter, '_getPointersFromEntry' ).callsArgWith(3, new Error('Failed to get token')); + + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response, function(err) { expect(err.message).to.equal('Failed to get token'); done(); @@ -3289,6 +3152,9 @@ describe('BucketsRouter', function() { '_getPointersFromEntry' ).callsArgWith(3, null, pointers); response.on('end', function() { + expect(bucketsRouter._createStorageEvents.callCount).to.equal(1); + expect(bucketsRouter._createStorageEvents.args[0][0]).to.equal(testUser); + expect(bucketsRouter._createStorageEvents.args[0][1]).to.equal(pointers); expect(bucketsRouter.storage.models.Bucket.findOne.args[0][0]) .to.eql({ _id: 'bucketid' }); expect(bucketsRouter._getPointersFromEntry.args[0][2]) @@ -3298,6 +3164,7 @@ describe('BucketsRouter', function() { ); done(); }); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response); }); @@ -3364,6 +3231,7 @@ describe('BucketsRouter', function() { ); done(); }); + sandbox.stub(bucketsRouter, '_createStorageEvents'); bucketsRouter.getFile(request, response); }); @@ -3978,7 +3846,7 @@ describe('BucketsRouter', function() { }); }); - it('should internal error if bucket entry not found', function(done) { + it('should give error if bucket entry query fails', function(done) { var request = httpMocks.createRequest({ method: 'DELETE', url: '/buckets/:bucket_id/files/:file_id', @@ -3992,36 +3860,51 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, {}); - var _bucketEntryFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' - ).returns({ - populate: function() { - return this; - }, - exec: sinon.stub().callsArgWith(0, new Error('Failed to lookup bucket entry')) + ).callsArgWith(1, new Error('test')); + bucketsRouter.removeFile(request, response, function(err) { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); + done(); }); - function StorageEvent() {} - StorageEvent.prototype.save = sinon.stub().callsArgWith(0, null); - var _storageEvent = sinon.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); + }); + + it('should give error if bucket entry not found', function(done) { + var request = httpMocks.createRequest({ + method: 'DELETE', + url: '/buckets/:bucket_id/files/:file_id', + params: { + id: 'bucketid', + file: 'fileid' + } + }); + request.user = someUser; + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + sandbox.stub( + bucketsRouter.storage.models.Bucket, + 'findOne' + ).callsArgWith(1, null, {}); + sandbox.stub( + bucketsRouter.storage.models.BucketEntry, + 'findOne' + ).callsArgWith(1, null, null); bucketsRouter.removeFile(request, response, function(err) { - _bucketFindOne.restore(); - _bucketEntryFindOne.restore(); - _storageEvent.restore(); - expect(err.message).to.equal('Failed to lookup bucket entry'); + expect(err).to.be.instanceOf(errors.NotFoundError); + expect(err.message).to.equal('File not found'); done(); }); }); - it('should not found error if bucket entry not found', function(done) { + it('should internal error if storage event deletion fails', function(done) { var request = httpMocks.createRequest({ method: 'DELETE', url: '/buckets/:bucket_id/files/:file_id', @@ -4035,31 +3918,25 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, {}); - var _bucketEntryFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' - ).returns({ - populate: function() { - return this; - }, - exec: sinon.stub().callsArgWith(0, null, null) + ).callsArgWith(1, null, { + remove: sandbox.stub().callsArgWith(0, null) }); - function StorageEvent() {} - StorageEvent.prototype.save = sinon.stub().callsArgWith(0, null); - var _storageEvent = sinon.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketEntryId') + .callsArgWith(2, null, hashes); + sandbox.stub( + bucketsRouter.storage.models.StorageEvent, + 'update' + ).callsArgWith(3, new Error('test')); bucketsRouter.removeFile(request, response, function(err) { - _bucketFindOne.restore(); - _bucketEntryFindOne.restore(); - _storageEvent.restore(); - expect(err.message).to.equal('File not found'); + expect(err.message).to.equal('test'); done(); }); }); @@ -4078,32 +3955,25 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, {}); - var _bucketEntryFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' - ).returns({ - populate: sinon.stub().returns({ - exec: sinon.stub().callsArgWith(0, null, { - remove: sinon.stub().callsArgWith(0, new Error('Failed to delete')) - }) - }) + ).callsArgWith(1, null, { + remove: sandbox.stub().callsArgWith(0, new Error('test')) }); - function StorageEvent() {} - StorageEvent.prototype.save = sinon.stub().callsArgWith(0, null); - var _storageEvent = sinon.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketEntryId') + .callsArgWith(2, null, hashes); + sandbox.stub( + bucketsRouter.storage.models.StorageEvent, + 'update' + ).callsArgWith(3, null); bucketsRouter.removeFile(request, response, function(err) { - _bucketFindOne.restore(); - _bucketEntryFindOne.restore(); - _storageEvent.restore(); - expect(err.message).to.equal('Failed to delete'); + expect(err.message).to.equal('test'); done(); }); }); @@ -4122,41 +3992,30 @@ describe('BucketsRouter', function() { req: request, eventEmitter: EventEmitter }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, {}); - var _bucketEntryFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' - ).returns({ - populate: sinon.stub().returns({ - exec: sinon.stub().callsArgWith(0, null, { - frame: { - size: 1000 - }, - remove: sinon.stub().callsArg(0) - }) - }) + ).callsArgWith(1, null, { + frame: { + size: 1000 + }, + remove: sinon.stub().callsArg(0) }); - function StorageEvent() {} - StorageEvent.prototype.save = sinon.stub().callsArgWith(0, null); - var _storageEvent = sinon.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketEntryId') + .callsArgWith(2, null, hashes); response.on('end', function() { - _bucketFindOne.restore(); - _bucketEntryFindOne.restore(); - _storageEvent.restore(); expect(response.statusCode).to.equal(204); done(); }); bucketsRouter.removeFile(request, response); }); -it('should throw error on storage event save failure', function(done) { + it('should give error on storage event update failure', function(done) { sandbox.stub(log, 'warn'); var request = httpMocks.createRequest({ method: 'DELETE', @@ -4171,40 +4030,32 @@ it('should throw error on storage event save failure', function(done) { req: request, eventEmitter: EventEmitter }); - var _bucketFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.Bucket, 'findOne' ).callsArgWith(1, null, {}); - var _bucketEntryFindOne = sinon.stub( + sandbox.stub( bucketsRouter.storage.models.BucketEntry, 'findOne' - ).returns({ - populate: sinon.stub().returns({ - exec: sinon.stub().callsArgWith(0, null, { - frame: { - size: 1000 - }, - remove: sinon.stub().callsArg(0) - }) - }) + ).callsArgWith(1, null, { + frame: { + size: 1000 + }, + remove: sandbox.stub().callsArg(0) }); - function StorageEvent() {} - StorageEvent.prototype.save = sinon.stub().callsArgWith(0, new Error('test')); - var _storageEvent = sinon.stub( - bucketsRouter.storage.models, - 'StorageEvent', - StorageEvent - ); - response.on('end', function() { - _bucketFindOne.restore(); - _bucketEntryFindOne.restore(); - _storageEvent.restore(); - expect(log.warn.callCount).to.equal(1); + const hashes = ['hash']; + sandbox.stub(bucketsRouter, '_getShardHashesByBucketEntryId') + .callsArgWith(2, null, hashes); + sandbox.stub( + bucketsRouter.storage.models.StorageEvent, + 'update' + ).callsArgWith(3, new Error('test')); + bucketsRouter.removeFile(request, response, function(err) { + expect(err).to.be.instanceOf(Error); + expect(err.message).to.equal('test'); done(); }); - bucketsRouter.removeFile(request, response); }); - }); describe('#getFileId', function() { diff --git a/test/server/routes/frames.unit.js b/test/server/routes/frames.unit.js index 01483619d..815693744 100644 --- a/test/server/routes/frames.unit.js +++ b/test/server/routes/frames.unit.js @@ -323,6 +323,51 @@ describe('FramesRouter', function() { }); + describe('#_createStorageEvent', function() { + const sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('will create storage event', function() { + const token = '688fedc83e7b0e9a5a7f68fa4e0098c7a40839c3'; + const user = { + _id: 'userid' + }; + const farmer = { + nodeID: 'c272cc90cf928328da80ba87c332bc97ed976118' + }; + const pointer = { + size: 1337, + hash: '25c7fbb6d7f0429a0a31ed91bdf8ce2ec2b51f11' + }; + var testFramesRouter = new FramesRouter( + require('../../_fixtures/router-opts') + ); + function StorageEvent(options) { + expect(options).to.eql({ + token: '688fedc83e7b0e9a5a7f68fa4e0098c7a40839c3', + user: 'userid', + client: 'userid', + farmer: 'c272cc90cf928328da80ba87c332bc97ed976118', + downloadBandwidth: 0, + storage: 1337, + timestamp: 1509178156367, + shardHash: '25c7fbb6d7f0429a0a31ed91bdf8ce2ec2b51f11', + success: false + }); + } + StorageEvent.prototype.save = sandbox.stub().callsArgWith(0, null); + testFramesRouter.storage = { + models: { + StorageEvent: StorageEvent + } + }; + const clock = sandbox.useFakeTimers(); + clock.tick(1509178156367); + testFramesRouter._createStorageEvent(token, user, farmer, pointer); + expect(StorageEvent.prototype.save.callCount).to.equal(1); + }); + }); + describe('#_getContractForShard', function() { const sandbox = sinon.sandbox.create(); afterEach(() => sandbox.restore()); @@ -1151,6 +1196,8 @@ describe('FramesRouter', function() { 'save' ).callsArgWith(0); + sandbox.stub(framesRouter, '_createStorageEvent'); + var frame1 = new framesRouter.storage.models.Frame({ user: someUser._id }); @@ -1169,33 +1216,37 @@ describe('FramesRouter', function() { ) }); - var _pointerCreate = sandbox.stub( - framesRouter.storage.models.Pointer, - 'create' - ).callsArgWith(1, null, new framesRouter.storage.models.Pointer({ + const pointer = new framesRouter.storage.models.Pointer({ index: 0, hash: storj.utils.rmd160('data'), size: 1024 * 1024 * 8, challenges: auditStream.getPrivateRecord().challenges, tree: auditStream.getPublicRecord() - })); + }); + var _pointerCreate = sandbox.stub( + framesRouter.storage.models.Pointer, + 'create' + ).callsArgWith(1, null, pointer); + + const farmer = storj.Contact({ + address: '127.0.0.1', + port: 1337, + nodeID: storj.utils.rmd160('farmer') + }); sandbox.stub( framesRouter, '_getContractForShard', function(contract, audit, bl, res, callback) { - callback(null, storj.Contact({ - address: '127.0.0.1', - port: 1337, - nodeID: storj.utils.rmd160('farmer') - }), contract); + callback(null, farmer, contract); } ); + const token = '16d1f480b1a59875d3e11251a0adaadef7817e88'; sandbox.stub( framesRouter.network, 'getConsignmentPointer' - ).callsArgWith(3, null, { token: 'token' }); + ).callsArgWith(3, null, { token: token }); response.on('end', function() { var result = response._getData(); @@ -1211,9 +1262,15 @@ describe('FramesRouter', function() { expect(result.farmer.nodeID).to.equal(storj.utils.rmd160('farmer')); expect(result.hash).to.equal(storj.utils.rmd160('data')); - expect(result.token).to.equal('token'); + expect(result.token).to.equal(token); expect(result.operation).to.equal('PUSH'); expect(testUser.recordUploadBytes.callCount).to.equal(1); + + expect(framesRouter._createStorageEvent.callCount).to.equal(1); + expect(framesRouter._createStorageEvent.args[0][0]).to.equal(token); + expect(framesRouter._createStorageEvent.args[0][1]).to.equal(testUser); + expect(framesRouter._createStorageEvent.args[0][2]).to.equal(farmer); + expect(framesRouter._createStorageEvent.args[0][3]).to.equal(pointer); done(); }); framesRouter.addShardToFrame(request, response); @@ -1302,6 +1359,8 @@ describe('FramesRouter', function() { } ]; + sandbox.stub(framesRouter, '_createStorageEvent'); + sandbox.stub( framesRouter.storage.models.Mirror, 'find' @@ -1462,6 +1521,8 @@ describe('FramesRouter', function() { } ]; + sandbox.stub(framesRouter, '_createStorageEvent'); + sandbox.stub( framesRouter.storage.models.Mirror, 'find' @@ -1622,6 +1683,8 @@ describe('FramesRouter', function() { ) }); + sandbox.stub(framesRouter, '_createStorageEvent'); + sandbox.stub( framesRouter.storage.models.Pointer, 'create' @@ -1838,6 +1901,8 @@ describe('FramesRouter', function() { }), }); + sandbox.stub(framesRouter, '_createStorageEvent'); + var frame1 = new framesRouter.storage.models.Frame({ user: someUser._id }); @@ -1956,6 +2021,8 @@ describe('FramesRouter', function() { }), }); + sandbox.stub(framesRouter, '_createStorageEvent'); + sandbox.stub( framesRouter.storage.models.Pointer, 'create' diff --git a/test/server/routes/reports.unit.js b/test/server/routes/reports.unit.js index b627aaae6..655f56fd3 100644 --- a/test/server/routes/reports.unit.js +++ b/test/server/routes/reports.unit.js @@ -8,28 +8,223 @@ const expect = require('chai').expect; const errors = require('storj-service-error-types'); const EventEmitter = require('events').EventEmitter; const ReportsRouter = require('../../../lib/server/routes/reports'); +const farmerMiddleware = require('../../../lib/server/middleware/farmer-auth'); describe('ReportsRouter', function() { - var reportsRouter = new ReportsRouter( require('../../_fixtures/router-opts') ); + describe('#authMiddleware', function() { + var sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('it will auth user with user auth headers', function(done) { + var request = httpMocks.createRequest({ + method: 'POST', + url: '/reports/exchanges', + body: {}, + headers: { + 'authorization': 'base64authstring' + } + }); + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + var testReportsRouter = new ReportsRouter( + require('../../_fixtures/router-opts') + ); + const bodyParser = sinon.stub().callsArgWith(2, null); + const userAuth = sinon.stub().callsArgWith(2, null); + testReportsRouter.userAuthMiddlewares = [ + bodyParser, + userAuth + ]; + testReportsRouter.authMiddleware(request, response, function(err) { + if (err) { + return done(err); + } + expect(bodyParser.callCount).to.equal(1); + expect(userAuth.callCount).to.equal(1); + done(); + }); + }); + it('it will auth farmer with farmer auth headers', function(done) { + var request = httpMocks.createRequest({ + method: 'POST', + url: '/reports/exchanges', + body: {}, + headers: { + 'x-node-id': '14fe443f9bfe4936fb70dd97298cc6a34c88cfba' + } + }); + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + sandbox.stub(farmerMiddleware, 'authFarmer').callsArgWith(2, null); + sandbox.stub(reportsRouter, 'farmerRawBodyMiddleware').callsArgWith(2, null); + reportsRouter.authMiddleware(request, response, function(err) { + if (err) { + return done(err); + } + expect(farmerMiddleware.authFarmer.callCount).to.equal(1); + done(); + }); + }); + it('it will error if no auth', function(done) { + var request = httpMocks.createRequest({ + method: 'POST', + url: '/reports/exchanges', + body: {}, + headers: {} + }); + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + reportsRouter.authMiddleware(request, response, function(err) { + expect(err).to.be.instanceOf(errors.NotAuthorizedError); + done(); + }); + }); + }); + + describe('#updateReputation', function() { + var sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + it('will record points and save', function() { + const contact = { + recordPoints: sandbox.stub().returns({ + save: sandbox.stub().callsArgWith(0, null) + }) + }; + sandbox.stub(reportsRouter.storage.models.Contact, 'findOne') + .callsArgWith(1, null, contact); + const nodeID = '2c5ae6807e9179cb2174d0265867c63abce48dfb'; + const points = 10; + reportsRouter.updateReputation(nodeID, points); + expect(reportsRouter.storage.models.Contact.findOne.callCount) + .to.equal(1); + expect(reportsRouter.storage.models.Contact.findOne.args[0][0]) + .to.eql({_id: nodeID}); + expect(contact.recordPoints.callCount).to.equal(1); + }); + + it('will return if contact not found', function() { + const contact = { + recordPoints: sandbox.stub().returns({ + save: sandbox.stub().callsArgWith(0, null) + }) + }; + sandbox.stub(reportsRouter.storage.models.Contact, 'findOne') + .callsArgWith(1, null, null); + const nodeID = '2c5ae6807e9179cb2174d0265867c63abce48dfb'; + const points = 10; + reportsRouter.updateReputation(nodeID, points); + expect(contact.recordPoints.callCount).to.equal(0); + }); + }); + + describe('#validateExchangeReport', function() { + var sandbox = sinon.sandbox.create(); + afterEach(() => sandbox.restore()); + + const validReports = [{ + token: '91e1fc2fd3a4c5244945e49c6f68ca1bd444d14c', + exchangeStart: 1509156812066, + exchangeEnd: 1509156822420, + exchangeResultCode: 1100, + exchangeResultMessage: 'FAILED_INTEGRITY' + }, { + token: 'fe081d837b4c6bbb0e416b8acd7b04ed29203f08', + exchangeStart: 1509156792819, + exchangeEnd: 1509156801731, + exchangeResultCode: 1000, + exchangeResultMessage: 'SHARD_DOWNLOADED' + }, { + token: 'a9d2c8cee65ad1b6ddb7cd574a18081f44ab8391', + exchangeStart: 1509156773796, + exchangeEnd: 1509156782011, + exchangeResultCode: 1100, + exchangeResultMessage: 'SHARD_UPLOADED' + }, { + token: 'b345adbe445452b6b451e4b8ca4beac2a548e22d', + exchangeStart: 1509156753155, + exchangeEnd: 1509156763347, + exchangeResultCode: 1000, + exchangeResultMessage: 'DOWNLOAD_ERROR' + }, { + token: '6563ac73bee62df44880da382cd352e3e2fe3374', + exchangeStart: 1509156731683, + exchangeEnd: 1509156742560, + exchangeResultCode: 1100, + exchangeResultMessage: 'TRANSFER_FAILED' + }, { + token: '518252128d9eb93b717618558ac64cf0bb882b36', + exchangeStart: 1509156731883, + exchangeEnd: 1509156732883, + exchangeResultCode: 1100, + exchangeResultMessage: 'MIRROR_SUCCESS' + }]; + + const invalidReports = [{ + token: '91e1fc2fd3a4c5244945e49c6f68ca1bd444d14c', + exchangeStart: 1509156812066, + exchangeEnd: 1509156822420, + exchangeResultCode: 1100, + exchangeResultMessage: 'NOT_A_VALID_MESSAGE' // invalid + }, { + token: 'fe081d837b4c6bbb0e416b8acd7b04ed29203f08', + exchangeStart: 'tuesday', // invalid + exchangeEnd: 1509156822421, + exchangeResultCode: 1000, + exchangeResultMessage: 'SHARD_DOWNLOADED' + }, { + token: 'fe081d837b4c6bbb0e416b8acd7b04ed29203f08', + exchangeStart: 1509156812068, + exchangeEnd: 'wednesday', // invalid + exchangeResultCode: 1000, + exchangeResultMessage: 'SHARD_DOWNLOADED' + }, { + token: 'a9d2c8cee65ad1b6ddb7cd574a18081f44ab8391', + exchangeStart: 1509156773796, + exchangeEnd: 1509156782011, + exchangeResultCode: 1234567890, // invalid + exchangeResultMessage: 'SHARD_UPLOADED' + }]; + + let i = 0; + validReports.forEach((report) => { + it('will validate report (' + i + ')', function() { + expect(reportsRouter.validateExchangeReport(report)).to.equal(true); + }); + i++; + }); + + invalidReports.forEach((report) => { + it('will invalidate report (' + i + ')', function() { + expect(reportsRouter.validateExchangeReport(report)).to.equal(false); + }); + i++; + }); + + }); + describe('#createExchangeReport', function() { var sandbox = sinon.sandbox.create(); afterEach(function() { sandbox.restore(); }); - it('should return internal error if save fails', function(done) { + it('should give internal error', function(done) { var request = httpMocks.createRequest({ method: 'POST', url: '/reports/exchanges', body: { - reporterId: storj.utils.rmd160('client'), - farmerId: storj.utils.rmd160('farmer'), - clientId: storj.utils.rmd160('client'), - dataHash: storj.utils.rmd160('data'), + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', exchangeStart: Date.now(), exchangeEnd: Date.now(), exchangeResultCode: 1000, @@ -41,30 +236,21 @@ describe('ReportsRouter', function() { eventEmitter: EventEmitter }); sandbox.stub( - reportsRouter.storage.models.Shard, - 'find' - ).callsArgWith(2, null, [{}]); - - sandbox.stub( - reportsRouter.storage.models.ExchangeReport.prototype, - 'save' - ).callsArgWith(0, new Error('Failed to save report')); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, new Error('test')); reportsRouter.createExchangeReport(request, response, function(err) { - expect(err).to.be.instanceOf(errors.BadRequestError); - expect(err.message).to.equal('Failed to save report'); + expect(err).to.be.instanceOf(errors.InternalError); done(); }); }); - it('should give error if datahash does not exist', function(done) { + it('should give not found without matching token', function(done) { var request = httpMocks.createRequest({ method: 'POST', url: '/reports/exchanges', body: { - reporterId: storj.utils.rmd160('client'), - farmerId: storj.utils.rmd160('farmer'), - clientId: storj.utils.rmd160('client'), - dataHash: storj.utils.rmd160('data'), + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', exchangeStart: Date.now(), exchangeEnd: Date.now(), exchangeResultCode: 1000, @@ -76,125 +262,266 @@ describe('ReportsRouter', function() { eventEmitter: EventEmitter }); sandbox.stub( - reportsRouter.storage.models.ExchangeReport.prototype, - 'save' - ).callsArgWith(0, null); - sandbox.stub( - reportsRouter.storage.models.Shard, - 'find' - ).callsArgWith(2, null, []); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, null); reportsRouter.createExchangeReport(request, response, function(err) { expect(err).to.be.instanceOf(errors.NotFoundError); - var save = reportsRouter.storage.models.ExchangeReport.prototype.save; - expect(save.callCount).to.equal(0); done(); }); }); - it('should give error if datahash does not exist', function(done) { + it('should give bad request error with invalid report', function(done) { var request = httpMocks.createRequest({ method: 'POST', url: '/reports/exchanges', body: { - reporterId: storj.utils.rmd160('client'), - farmerId: storj.utils.rmd160('farmer'), - clientId: storj.utils.rmd160('client'), - dataHash: storj.utils.rmd160('data'), + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', exchangeStart: Date.now(), exchangeEnd: Date.now(), - exchangeResultCode: 1000, - exchangeResultMessage: 'SUCCESS' + exchangeResultCode: 1234567890, + exchangeResultMessage: 'NOT_A_MESSAGE' } }); var response = httpMocks.createResponse({ req: request, eventEmitter: EventEmitter }); + const event = {}; sandbox.stub( - reportsRouter.storage.models.ExchangeReport.prototype, - 'save' - ).callsArgWith(0, null); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, event); + reportsRouter.createExchangeReport(request, response, function(err) { + expect(err).to.be.instanceOf(errors.BadRequestError); + done(); + }); + }); + + it('should give not authorized if not valid reporter', function(done) { + var request = httpMocks.createRequest({ + method: 'POST', + url: '/reports/exchanges', + body: { + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', + exchangeStart: Date.now(), + exchangeEnd: Date.now(), + exchangeResultCode: 1000, + exchangeResultMessage: 'SHARD_DOWNLOADED' + } + }); + request.user = { + id: 'userid1' + }; + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + const event = { + client: 'userid2' + }; sandbox.stub( - reportsRouter.storage.models.Shard, - 'find' - ).callsArgWith(2, null, null); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, event); reportsRouter.createExchangeReport(request, response, function(err) { - expect(err).to.be.instanceOf(errors.NotFoundError); - var save = reportsRouter.storage.models.ExchangeReport.prototype.save; - expect(save.callCount).to.equal(0); + expect(err).to.be.instanceOf(errors.NotAuthorizedError); done(); }); }); - it('should give error if datahash does not exist', function(done) { + it('will update storage event with client (success)', function(done) { var request = httpMocks.createRequest({ method: 'POST', url: '/reports/exchanges', body: { - reporterId: storj.utils.rmd160('client'), - farmerId: storj.utils.rmd160('farmer'), - clientId: storj.utils.rmd160('client'), - dataHash: storj.utils.rmd160('data'), + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', exchangeStart: Date.now(), exchangeEnd: Date.now(), exchangeResultCode: 1000, - exchangeResultMessage: 'SUCCESS' + exchangeResultMessage: 'SHARD_DOWNLOADED' } }); + request.user = { + id: 'userid1' + }; var response = httpMocks.createResponse({ req: request, eventEmitter: EventEmitter }); + const event = { + client: 'userid1', + farmer: 'nodeid', + save: sandbox.stub().callsArgWith(0, null) + }; + sandbox.stub(reportsRouter, '_handleExchangeReport'); + sandbox.stub(reportsRouter, 'updateReputation'); sandbox.stub( - reportsRouter.storage.models.Shard, - 'find' - ).callsArgWith(2, new Error('Internal error')); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, event); + response.on('end', function() { + expect(reportsRouter.updateReputation.callCount).to.equal(1); + expect(reportsRouter.updateReputation.args[0][0]).to.equal('nodeid'); + expect(reportsRouter.updateReputation.args[0][1]).to.equal(10); + expect(event.save.callCount).to.equal(1); + expect(response.statusCode).to.equal(201); + done(); + }); + reportsRouter.createExchangeReport(request, response); + }); + + it('will update storage event with client (failure)', function(done) { + var request = httpMocks.createRequest({ + method: 'POST', + url: '/reports/exchanges', + body: { + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', + exchangeStart: Date.now(), + exchangeEnd: Date.now(), + exchangeResultCode: 1100, + exchangeResultMessage: 'SHARD_DOWNLOADED' + } + }); + request.user = { + id: 'userid1' + }; + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + const event = { + client: 'userid1', + farmer: 'nodeid', + save: sandbox.stub().callsArgWith(0, null) + }; + sandbox.stub(reportsRouter, '_handleExchangeReport'); + sandbox.stub(reportsRouter, 'updateReputation'); sandbox.stub( - reportsRouter.storage.models.ExchangeReport.prototype, - 'save' - ).callsArgWith(0, null); - reportsRouter.createExchangeReport(request, response, function(err) { - expect(err).to.be.instanceOf(errors.InternalError); - var save = reportsRouter.storage.models.ExchangeReport.prototype.save; - expect(save.callCount).to.equal(0); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, event); + response.on('end', function() { + expect(reportsRouter.updateReputation.callCount).to.equal(1); + expect(reportsRouter.updateReputation.args[0][0]).to.equal('nodeid'); + expect(reportsRouter.updateReputation.args[0][1]).to.equal(-10); + expect(event.save.callCount).to.equal(1); + expect(response.statusCode).to.equal(201); done(); }); + reportsRouter.createExchangeReport(request, response); }); - it('should send 201 if report saved', function(done) { + it('will update storage event with client (idempotence)', function(done) { var request = httpMocks.createRequest({ method: 'POST', url: '/reports/exchanges', body: { - reporterId: storj.utils.rmd160('client'), - farmerId: storj.utils.rmd160('farmer'), - clientId: storj.utils.rmd160('client'), - dataHash: storj.utils.rmd160('data'), + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', exchangeStart: Date.now(), exchangeEnd: Date.now(), exchangeResultCode: 1000, - exchangeResultMessage: 'SUCCESS' + exchangeResultMessage: 'SHARD_DOWNLOADED' } }); + request.user = { + id: 'userid1' + }; var response = httpMocks.createResponse({ req: request, eventEmitter: EventEmitter }); + const event = { + client: 'userid1', + save: sandbox.stub().callsArgWith(0, null), + clientReport: { + exchangeResultCode: 1100 // already has report + } + }; + sandbox.stub(reportsRouter, '_handleExchangeReport'); sandbox.stub( - reportsRouter.storage.models.Shard, - 'find' - ).callsArgWith(2, null, [{}]); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, event); + response.on('end', function() { + expect(event.save.callCount).to.equal(0); + expect(response.statusCode).to.equal(200); + done(); + }); + reportsRouter.createExchangeReport(request, response); + }); + + it('will update storage event with farmer', function(done) { + var request = httpMocks.createRequest({ + method: 'POST', + url: '/reports/exchanges', + body: { + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', + exchangeStart: Date.now(), + exchangeEnd: Date.now(), + exchangeResultCode: 1000, + exchangeResultMessage: 'SHARD_DOWNLOADED' + } + }); + request.farmerNodeID = '4b449e6445daf4bfe0e7add6ca10bd66e27e1663'; + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + const event = { + farmer: '4b449e6445daf4bfe0e7add6ca10bd66e27e1663', + save: sandbox.stub().callsArgWith(0, null) + }; + sandbox.stub(reportsRouter, '_handleExchangeReport'); sandbox.stub( - reportsRouter.storage.models.ExchangeReport.prototype, - 'save' - ).callsArgWith(0, null); + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, event); response.on('end', function() { + expect(event.save.callCount).to.equal(1); expect(response.statusCode).to.equal(201); done(); }); reportsRouter.createExchangeReport(request, response); }); + it('will update storage event with farmer (idempotence)', function(done) { + var request = httpMocks.createRequest({ + method: 'POST', + url: '/reports/exchanges', + body: { + token: 'f4c0fcfcc818e162c39b9b678a54124c847c0f9a', + exchangeStart: Date.now(), + exchangeEnd: Date.now(), + exchangeResultCode: 1000, + exchangeResultMessage: 'SHARD_DOWNLOADED' + } + }); + request.farmerNodeID = '4b449e6445daf4bfe0e7add6ca10bd66e27e1663'; + var response = httpMocks.createResponse({ + req: request, + eventEmitter: EventEmitter + }); + const event = { + farmer: '4b449e6445daf4bfe0e7add6ca10bd66e27e1663', + save: sandbox.stub().callsArgWith(0, null), + farmerReport: { + exchangeResultCode: 1100 // already has report + } + }; + sandbox.stub(reportsRouter, '_handleExchangeReport'); + sandbox.stub( + reportsRouter.storage.models.StorageEvent, + 'findOne' + ).callsArgWith(1, null, event); + response.on('end', function() { + expect(event.save.callCount).to.equal(0); + expect(response.statusCode).to.equal(200); + done(); + }); + reportsRouter.createExchangeReport(request, response); + }); + }); describe('#_handleExchangeReport', function() { @@ -205,15 +532,16 @@ describe('ReportsRouter', function() { _triggerMirrorEstablish = sinon.stub( reportsRouter, '_triggerMirrorEstablish' - ).callsArg(2); + ).callsArg(3); }); after(() => _triggerMirrorEstablish.restore()); it('should callback error if not valid report type', function(done) { + const event = {}; reportsRouter._handleExchangeReport({ shardHash: 'hash', exchangeResultMessage: 'NOT_VALID' - }, (err) => { + }, event, (err) => { expect(err.message).to.equal( 'Exchange result type will not trigger action' ); @@ -222,24 +550,27 @@ describe('ReportsRouter', function() { }); it('should trigger a mirror on SHARD_UPLOADED', function(done) { + const event = {}; reportsRouter._handleExchangeReport({ shardHash: 'hash', exchangeResultMessage: 'SHARD_UPLOADED' - }, done); + }, event, done); }); it('should trigger a mirror on MIRROR_SUCCESS', function(done) { + const event = {}; reportsRouter._handleExchangeReport({ shardHash: 'hash', exchangeResultMessage: 'MIRROR_SUCCESS' - }, done); + }, event, done); }); it('should trigger a mirror on DOWNLOAD_ERROR', function(done) { + const event = {}; reportsRouter._handleExchangeReport({ shardHash: 'hash', exchangeResultMessage: 'DOWNLOAD_ERROR' - }, done); + }, event, done); }); }); @@ -407,7 +738,11 @@ describe('ReportsRouter', function() { sandbox.stub( reportsRouter.network, 'getRetrievalPointer' - ).callsArgWith(2, null, { /* pointer */ }); + ).callsArgWith(2, null, { + farmer: { + nodeID: '9fbe85050ecf276e3f47a979cb33bc55172ad241' + } + }); sandbox.stub( reportsRouter.network, 'getMirrorNodes' @@ -416,7 +751,9 @@ describe('ReportsRouter', function() { reportsRouter.contracts, 'save' ).callsArgWith(1, null); - reportsRouter._triggerMirrorEstablish(n, hash, function(err) { + const event = {}; + sandbox.stub(reportsRouter, '_createStorageEvent'); + reportsRouter._triggerMirrorEstablish(n, hash, event, function(err) { expect(err).to.equal(null); expect(Array.prototype.sort.callCount).to.equal(1); expect(Array.prototype.sort.args[0][0]) @@ -426,7 +763,7 @@ describe('ReportsRouter', function() { }); it('should error if net mirroring fails', function(done) { - var _mirrorFind = sinon.stub( + sandbox.stub( reportsRouter.storage.models.Mirror, 'find' ).returns({ @@ -475,11 +812,11 @@ describe('ReportsRouter', function() { } } }); - var _contractsLoad = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'load' ).callsArgWith(1, null, item); - var _getContactById = sinon.stub( + sandbox.stub( reportsRouter, 'getContactById' ).callsArgWith(1, null, new reportsRouter.storage.models.Contact({ @@ -490,32 +827,32 @@ describe('ReportsRouter', function() { lastSeen: Date.now(), userAgent: 'test' })); - var _getRetrievalPointer = sinon.stub( + sandbox.stub( reportsRouter.network, 'getRetrievalPointer' - ).callsArgWith(2, null, { /* pointer */ }); - var _getMirrorNodes = sinon.stub( + ).callsArgWith(2, null, { + farmer: { + nodeID: '9fbe85050ecf276e3f47a979cb33bc55172ad241' + } + }); + sandbox.stub( reportsRouter.network, 'getMirrorNodes' ).callsArgWith(2, new Error('Failed to mirror data')); - var _contractsSave = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'save' ).callsArgWith(1, null); - reportsRouter._triggerMirrorEstablish(n, hash, function(err) { - _mirrorFind.restore(); - _contractsLoad.restore(); - _getContactById.restore(); - _getRetrievalPointer.restore(); - _getMirrorNodes.restore(); - _contractsSave.restore(); + const event = {}; + sandbox.stub(reportsRouter, '_createStorageEvent'); + reportsRouter._triggerMirrorEstablish(n, hash, event, function(err) { expect(err.message).to.equal('Failed to mirror data'); done(); }); }); it('should error if no pointer can be retrieved', function(done) { - var _mirrorFind = sinon.stub( + sandbox.stub( reportsRouter.storage.models.Mirror, 'find' ).returns({ @@ -564,11 +901,11 @@ describe('ReportsRouter', function() { } } }); - var _contractsLoad = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'load' ).callsArgWith(1, null, item); - var _getContactById = sinon.stub( + sandbox.stub( reportsRouter, 'getContactById' ).callsArgWith(1, null, new reportsRouter.storage.models.Contact({ @@ -579,32 +916,28 @@ describe('ReportsRouter', function() { lastSeen: Date.now(), userAgent: 'test' })); - var _getRetrievalPointer = sinon.stub( + sandbox.stub( reportsRouter.network, 'getRetrievalPointer' ).callsArgWith(2, new Error('Failed to retrieve pointer')); - var _getMirrorNodes = sinon.stub( + sandbox.stub( reportsRouter.network, 'getMirrorNodes' ).callsArgWith(2, null); - var _contractsSave = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'save' ).callsArgWith(1, null); - reportsRouter._triggerMirrorEstablish(n, hash, function(err) { - _mirrorFind.restore(); - _contractsLoad.restore(); - _getContactById.restore(); - _getRetrievalPointer.restore(); - _getMirrorNodes.restore(); - _contractsSave.restore(); + const event = {}; + sandbox.stub(reportsRouter, '_createStorageEvent'); + reportsRouter._triggerMirrorEstablish(n, hash, event, function(err) { expect(err.message).to.equal('Failed to get pointer'); done(); }); }); it('should error if no pointer can be retrieved', function(done) { - var _mirrorFind = sinon.stub( + sandbox.stub( reportsRouter.storage.models.Mirror, 'find' ).returns({ @@ -653,39 +986,35 @@ describe('ReportsRouter', function() { } } }); - var _contractsLoad = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'load' ).callsArgWith(1, null, item); - var _getContactById = sinon.stub( + sandbox.stub( reportsRouter, 'getContactById' ).callsArgWith(1, new Error('Contact not found')); - var _getRetrievalPointer = sinon.stub( + sandbox.stub( reportsRouter.network, 'getRetrievalPointer' ).callsArgWith(2, null, { /* pointer */ }); - var _getMirrorNodes = sinon.stub( + sandbox.stub( reportsRouter.network, 'getMirrorNodes' ).callsArgWith(2, null); - var _contractsSave = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'save' ).callsArgWith(1, null); - reportsRouter._triggerMirrorEstablish(n, hash, function(err) { - _mirrorFind.restore(); - _contractsLoad.restore(); - _getContactById.restore(); - _getRetrievalPointer.restore(); - _getMirrorNodes.restore(); - _contractsSave.restore(); + const event = {}; + sandbox.stub(reportsRouter, '_createStorageEvent'); + reportsRouter._triggerMirrorEstablish(n, hash, event, function(err) { expect(err.message).to.equal('Failed to get pointer'); done(); }); }); it('should error if the contract cannot load', function(done) { - var _mirrorFind = sinon.stub( + sandbox.stub( reportsRouter.storage.models.Mirror, 'find' ).returns({ @@ -726,11 +1055,11 @@ describe('ReportsRouter', function() { }; } }); - var _contractsLoad = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'load' ).callsArgWith(1, new Error('Failed to load contract')); - var _getContactById = sinon.stub( + sandbox.stub( reportsRouter, 'getContactById' ).callsArgWith(1, null, new reportsRouter.storage.models.Contact({ @@ -741,32 +1070,28 @@ describe('ReportsRouter', function() { lastSeen: Date.now(), userAgent: 'test' })); - var _getRetrievalPointer = sinon.stub( + sandbox.stub( reportsRouter.network, 'getRetrievalPointer' ).callsArgWith(2, null, { /* pointer */ }); - var _getMirrorNodes = sinon.stub( + sandbox.stub( reportsRouter.network, 'getMirrorNodes' ).callsArgWith(2, null); - var _contractsSave = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'save' ).callsArgWith(1, null); - reportsRouter._triggerMirrorEstablish(n, hash, function(err) { - _mirrorFind.restore(); - _contractsLoad.restore(); - _getContactById.restore(); - _getRetrievalPointer.restore(); - _getMirrorNodes.restore(); - _contractsSave.restore(); + const event = {}; + sandbox.stub(reportsRouter, '_createStorageEvent'); + reportsRouter._triggerMirrorEstablish(n, hash, event, function(err) { expect(err.message).to.equal('Failed to load contract'); done(); }); }); it('should error if the mirror limit is reached', function(done) { - var _mirrorFind = sinon.stub( + sandbox.stub( reportsRouter.storage.models.Mirror, 'find' ).returns({ @@ -831,11 +1156,11 @@ describe('ReportsRouter', function() { } } }); - var _contractsLoad = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'load' ).callsArgWith(1, null, item); - var _getContactById = sinon.stub( + sandbox.stub( reportsRouter, 'getContactById' ).callsArgWith(1, null, new reportsRouter.storage.models.Contact({ @@ -846,32 +1171,28 @@ describe('ReportsRouter', function() { lastSeen: Date.now(), userAgent: 'test' })); - var _getRetrievalPointer = sinon.stub( + sandbox.stub( reportsRouter.network, 'getRetrievalPointer' ).callsArgWith(2, null, { /* pointer */ }); - var _getMirrorNodes = sinon.stub( + sandbox.stub( reportsRouter.network, 'getMirrorNodes' ).callsArgWith(2, null); - var _contractsSave = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'save' ).callsArgWith(1, null); - reportsRouter._triggerMirrorEstablish(2, hash, function(err) { - _mirrorFind.restore(); - _contractsLoad.restore(); - _getContactById.restore(); - _getRetrievalPointer.restore(); - _getMirrorNodes.restore(); - _contractsSave.restore(); + const event = {}; + sandbox.stub(reportsRouter, '_createStorageEvent'); + reportsRouter._triggerMirrorEstablish(2, hash, event, function(err) { expect(err.message).to.equal('Auto mirroring limit is reached'); done(); }); }); it('should error if no mirrors are available', function(done) { - var _mirrorFind = sinon.stub( + sandbox.stub( reportsRouter.storage.models.Mirror, 'find' ).returns({ @@ -889,11 +1210,11 @@ describe('ReportsRouter', function() { } } }); - var _contractsLoad = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'load' ).callsArgWith(1, null, item); - var _getContactById = sinon.stub( + sandbox.stub( reportsRouter, 'getContactById' ).callsArgWith(1, null, new reportsRouter.storage.models.Contact({ @@ -904,25 +1225,21 @@ describe('ReportsRouter', function() { lastSeen: Date.now(), userAgent: 'test' })); - var _getRetrievalPointer = sinon.stub( + sandbox.stub( reportsRouter.network, 'getRetrievalPointer' ).callsArgWith(2, null, { /* pointer */ }); - var _getMirrorNodes = sinon.stub( + sandbox.stub( reportsRouter.network, 'getMirrorNodes' ).callsArgWith(2, null); - var _contractsSave = sinon.stub( + sandbox.stub( reportsRouter.contracts, 'save' ).callsArgWith(1, null); - reportsRouter._triggerMirrorEstablish(n, hash, function(err) { - _mirrorFind.restore(); - _contractsLoad.restore(); - _getContactById.restore(); - _getRetrievalPointer.restore(); - _getMirrorNodes.restore(); - _contractsSave.restore(); + const event = {}; + sandbox.stub(reportsRouter, '_createStorageEvent'); + reportsRouter._triggerMirrorEstablish(n, hash, event, function(err) { expect(err.message).to.equal('No available mirrors'); done(); });