From 42333308137bb654b17b704ef7b24f81bd5939a3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 19 Oct 2017 14:31:05 -0400 Subject: [PATCH 01/11] Sketch out new model --- lib/models/storage-event.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index 325252ad..9e44b5a7 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -1,5 +1,23 @@ 'use strict'; +// UPDATE WITH THESE +// - token (id) - unique identifier for this event +// - bucket (id) +// - bucketEntry (id) +// - client (user-id) +// - farmer (farmer-id) +// - timestamp (number) +// - downloadBandwidth (number) +// - storage (number) +// - shardHash (id) +// - success (boolean) +// - senderReport/receiverReport +// - exchangeStart +// - exchangeEnd +// - exchangeResultCode +// - exchangeResultMessage +// - storeEnd + const mongoose = require('mongoose'); const utils = require('../utils'); /** From 6b20702b4597ed8305ab19f51b05cb658480fc43 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 19 Oct 2017 16:16:05 -0400 Subject: [PATCH 02/11] Add to model --- lib/models/storage-event.js | 89 ++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index 9e44b5a7..57cdd58f 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -1,32 +1,19 @@ 'use strict'; -// UPDATE WITH THESE -// - token (id) - unique identifier for this event -// - bucket (id) -// - bucketEntry (id) -// - client (user-id) -// - farmer (farmer-id) -// - timestamp (number) -// - downloadBandwidth (number) -// - storage (number) -// - shardHash (id) -// - success (boolean) -// - senderReport/receiverReport -// - exchangeStart -// - exchangeEnd -// - exchangeResultCode -// - exchangeResultMessage -// - storeEnd - const mongoose = require('mongoose'); const utils = require('../utils'); /** - * Represents the history of storage related summary + * Represents the history of storage related summary * statistics for buckets and their associated files * @constructor */ var StorageEvent = new mongoose.Schema({ + token: { + type: String, + required: false, + unique: true + }, bucket: { type: mongoose.Schema.Types.ObjectId, ref: 'Bucket', @@ -37,8 +24,11 @@ var StorageEvent = new mongoose.Schema({ ref: 'BucketEntry', required: true }, + // User can be both a user in the database as well + // as a farmer in the case where storage events are created + // during mirrors. user: { - type: String, + type: String, ref: 'User', required: true, validate: { @@ -46,18 +36,75 @@ var StorageEvent = new mongoose.Schema({ message: 'Invalid user email address' } }, + farmer: { + type: String, + required: true + } timestamp: { type: Date, default: Date.now, required: true }, + storeEnd: { + type: Date, + required: false + }, downloadBandwidth: { type: Number, required: false }, storage: { - type: Number, + type: Number, required: false + }, + farmerStorage: { + type: Number, + required: false + }, + shardHash: { + type: String, + required: false + }, + userReport: { + exchangeStart: { + type: Date, + required: false + }, + exchangeEnd: { + type: Date, + required: false + }, + exchangeResultCode: { + type: Number, + required: false + }, + exchangeResultMessage: { + type: String, + required: false + } + }, + farmerReport: { + exchangeStart: { + type: Date, + required: false + }, + exchangeEnd: { + type: Date, + required: false + }, + exchangeResultCode: { + type: Number, + required: false + }, + exchangeResultMessage: { + type: String, + required: false + } + }, + success: { + type: Boolean, + required: true, + default: false } }); From fa8ef18c73caaed822a105148c4c7c735cfb4cb7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 23 Oct 2017 15:38:13 -0400 Subject: [PATCH 03/11] Switch to reference based on frame --- lib/models/storage-event.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index 57cdd58f..f3cc4814 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -14,14 +14,9 @@ var StorageEvent = new mongoose.Schema({ required: false, unique: true }, - bucket: { + frame: { type: mongoose.Schema.Types.ObjectId, - ref: 'Bucket', - required: true - }, - bucketEntry: { - type: mongoose.Schema.Types.ObjectId, - ref: 'BucketEntry', + ref: 'Frame', required: true }, // User can be both a user in the database as well From 6d8aed966f82cea4934b5d1546d1ce49b2fa1527 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 Oct 2017 17:50:55 -0400 Subject: [PATCH 04/11] Update model - Have users be an array. Mirror storage events of a shard that multiple users have uploaded will have multiple users interested. This will provide an ability to track how long the mirror should be held. When one user deletes the shard, the user will be removed from the storage event. When there is only one user the storeEnd will be updated to terminate the storage of the shard. - Seperate user and client, thus it's possible to have a client and user for mirror actions, so that it's possible to query storage events of mirrors by a user. --- lib/models/storage-event.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index f3cc4814..9be6bb77 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -14,15 +14,10 @@ var StorageEvent = new mongoose.Schema({ required: false, unique: true }, - frame: { - type: mongoose.Schema.Types.ObjectId, - ref: 'Frame', - required: true - }, // User can be both a user in the database as well // as a farmer in the case where storage events are created // during mirrors. - user: { + users: [{ type: String, ref: 'User', required: true, @@ -30,7 +25,11 @@ var StorageEvent = new mongoose.Schema({ validator: value => utils.isValidEmail(value), message: 'Invalid user email address' } - }, + }], + client: { + type: String, + required: true + } farmer: { type: String, required: true @@ -60,7 +59,7 @@ var StorageEvent = new mongoose.Schema({ type: String, required: false }, - userReport: { + clientReport: { exchangeStart: { type: Date, required: false From d3178d6d460f7e31dd0908e76456f823ea400fed Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 Oct 2017 19:08:05 -0400 Subject: [PATCH 05/11] Handle mirroring case - Mirrors will be created without a user and only a client - Queries can fold all storage events based on the hash and determine all users and storage time frames --- lib/models/storage-event.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index 9be6bb77..3e27b552 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -14,10 +14,7 @@ var StorageEvent = new mongoose.Schema({ required: false, unique: true }, - // User can be both a user in the database as well - // as a farmer in the case where storage events are created - // during mirrors. - users: [{ + user: { type: String, ref: 'User', required: true, @@ -25,7 +22,7 @@ var StorageEvent = new mongoose.Schema({ validator: value => utils.isValidEmail(value), message: 'Invalid user email address' } - }], + }, client: { type: String, required: true @@ -39,10 +36,14 @@ var StorageEvent = new mongoose.Schema({ default: Date.now, required: true }, - storeEnd: { + farmerEnd: { type: Date, required: false }, + userDeleted: { + type: Date, + required: false + } downloadBandwidth: { type: Number, required: false From 4e7c4dbc4b18b3ef69e4d5c30cd5fef6e5a91e98 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 24 Oct 2017 19:14:18 -0400 Subject: [PATCH 06/11] Remove duplicate data --- lib/models/storage-event.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index 3e27b552..06d8aa94 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -52,10 +52,6 @@ var StorageEvent = new mongoose.Schema({ type: Number, required: false }, - farmerStorage: { - type: Number, - required: false - }, shardHash: { type: String, required: false From 21a02966e32fb5392f64f2bda8326f468ee2f46d Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Sat, 28 Oct 2017 02:55:09 -0400 Subject: [PATCH 07/11] Fix syntax errors, and update tests --- lib/models/storage-event.js | 6 +++--- test/storage-event.unit.js | 23 ++++++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index 06d8aa94..f9299876 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -26,11 +26,11 @@ var StorageEvent = new mongoose.Schema({ client: { type: String, required: true - } + }, farmer: { type: String, required: true - } + }, timestamp: { type: Date, default: Date.now, @@ -43,7 +43,7 @@ var StorageEvent = new mongoose.Schema({ userDeleted: { type: Date, required: false - } + }, downloadBandwidth: { type: Number, required: false diff --git a/test/storage-event.unit.js b/test/storage-event.unit.js index cc1f135c..3dda0379 100644 --- a/test/storage-event.unit.js +++ b/test/storage-event.unit.js @@ -30,22 +30,27 @@ describe('Storage/models/Storage-Event', function() { it('should create storage event with default props', function(done) { var newStorageEvent = new StorageEvent({ - bucket: mongoose.Types.ObjectId(), - bucketEntry: mongoose.Types.ObjectId(), user: 'user@gmail.com', downloadBandwidth: 0, storage: 1000000, + farmer: 'c5857d99d3ff951701093b75fba94e1c82877f9b', + client: 'user@gmail.com' }); - newStorageEvent.save(function(err, storEvent) { + newStorageEvent.save(function(err, storeEvent) { + if (err) { + return done(err); + } let ObjectIdType = mongoose.Types.ObjectId; expect(err).to.not.be.an.instanceOf(Error); - expect(storEvent.bucket).to.be.an.instanceOf(ObjectIdType); - expect(storEvent.bucketEntry).to.be.an.instanceOf(ObjectIdType); - expect(storEvent.user).to.be.a('string'); - expect(storEvent.timestamp).to.be.an.instanceOf(Date); - expect(storEvent.downloadBandwidth).to.be.a('number'); - expect(storEvent.storage).to.be.a('number'); + expect(storeEvent._id).to.be.instanceOf(ObjectIdType); + expect(storeEvent.user).to.be.a('string'); + expect(storeEvent.timestamp).to.be.an.instanceOf(Date); + expect(storeEvent.downloadBandwidth).to.be.a('number'); + expect(storeEvent.storage).to.be.a('number'); + expect(storeEvent.farmer) + .to.equal('c5857d99d3ff951701093b75fba94e1c82877f9b'); + expect(storeEvent.client).to.equal('user@gmail.com'); done(); }); }); From ebdbbd39ae7a983089f390d48a510ef3244143d8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Sat, 28 Oct 2017 04:20:34 -0400 Subject: [PATCH 08/11] User not required --- lib/models/storage-event.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index f9299876..cafab3da 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -17,7 +17,7 @@ var StorageEvent = new mongoose.Schema({ user: { type: String, ref: 'User', - required: true, + required: false, validate: { validator: value => utils.isValidEmail(value), message: 'Invalid user email address' From 0be50cbd6b52fc69f9dd13f51102b7669ae4629e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Sat, 28 Oct 2017 04:25:13 -0400 Subject: [PATCH 09/11] Validate false values: null, undefined as valid --- lib/models/storage-event.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/storage-event.js b/lib/models/storage-event.js index cafab3da..e3318bec 100644 --- a/lib/models/storage-event.js +++ b/lib/models/storage-event.js @@ -19,7 +19,7 @@ var StorageEvent = new mongoose.Schema({ ref: 'User', required: false, validate: { - validator: value => utils.isValidEmail(value), + validator: value => utils.isValidEmail(value) || !value, message: 'Invalid user email address' } }, From 411a187e76fe5db6464c7f9cb494e40e4c1fd630 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 13 Nov 2017 13:54:02 -0500 Subject: [PATCH 10/11] Add cron model --- lib/models/cron.js | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 lib/models/cron.js diff --git a/lib/models/cron.js b/lib/models/cron.js new file mode 100644 index 00000000..35cffed2 --- /dev/null +++ b/lib/models/cron.js @@ -0,0 +1,48 @@ +'use strict'; + +const mongoose = require('mongoose'); +const SchemaOptions = require('../options'); +const errors = require('storj-service-error-types'); + +const CronJob = new mongoose.Schema({ + name: { + type: String, + required: true, + unique: true, + index: true + }, + locked: { + type: Boolean, + default: false + required: true + }, + started: { + type: Date, + default: Date.now, + required: true + }, + data: { + type: String, + required: false + } +}); + +Partner.set('toObject', { + transform: function(doc, ret) { + ret.id = ret._id; + delete ret.__v; + delete ret._id; + } +}); + +Partner.set('toJSON', { + transform: function(doc, ret) { + ret.id = ret._id; + delete ret.__v; + delete ret._id; + } +}); + +module.exports = function(connection) { + return connection.model('CronJob', CronJob); +}; From 51955c76a2de19c334a3648296d6b9ad1c175cf3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 13 Nov 2017 14:09:47 -0500 Subject: [PATCH 11/11] Add finished time --- lib/models/cron.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/models/cron.js b/lib/models/cron.js index 35cffed2..73e2ebff 100644 --- a/lib/models/cron.js +++ b/lib/models/cron.js @@ -21,6 +21,11 @@ const CronJob = new mongoose.Schema({ default: Date.now, required: true }, + finished: { + type: Date, + default: Date.now, + required: true + }, data: { type: String, required: false