Skip to content

Commit

Permalink
Bundle related functions moved from EntityBuilder to BundleBuilder; E…
Browse files Browse the repository at this point in the history
…ntityDownloader changed name to BundleDownloader; Bundle repository extracted out of entity repository; Removed some unused methods and updated documentation; proofBlock renamed to bundleProofBlock; (#312)
  • Loading branch information
spherefoundry committed Dec 12, 2018
1 parent 361efac commit 06a1da0
Show file tree
Hide file tree
Showing 17 changed files with 724 additions and 666 deletions.
21 changes: 14 additions & 7 deletions apiary.apib
Original file line number Diff line number Diff line change
Expand Up @@ -612,7 +612,8 @@ Fetches asset with provided assetId in the system.
+ bundleId (string) - If the asset has already been added to a bundle, Id of the bundle holding the event
+ bundleTransactionHash (string) - If the asset has already been added to a bundle, identifier of the transaction on which the proof of bundle has been uploaded.
More info on the transaction can be found under `http://explorer-test.ambrosus.com/#/tx/{bundleTransactionHash}`

+ bundleProofBlock (number) - If the asset has already been added to a bundle, block number in which the proof of the bundle has been included
+ bundleUploadTimestamp (number) - If the asset has already been added to a bundle, timestamp of bundle proof upload


+ Body
Expand All @@ -630,6 +631,8 @@ Fetches asset with provided assetId in the system.
"metadata": {
"bundleId": "0x85a427a3.....cd1d38ebbd",
"bundleTransactionHash": "0x21ab....1cdf8e55b37"
"bundleUploadTimestamp": 1503424969,
"bundleProofBlock": 3221
}
}

Expand Down Expand Up @@ -676,6 +679,8 @@ Finds assets satisfying provided criteria.
+ bundleId (string) - If the asset has already been added to a bundle, Id of the bundle holding the event
+ bundleTransactionHash (string) - If the asset has already been added to a bundle, identifier of the transaction on which the proof of bundle has been uploaded.
More info on the transaction can be found under `http://explorer-test.ambrosus.com/#/tx/{bundleTransactionHash}`
+ bundleProofBlock (number) - If the asset has already been added to a bundle, block number in which the proof of the bundle has been included
+ bundleUploadTimestamp (number) - If the asset has already been added to a bundle, timestamp of bundle proof upload

+ resultCount (number) - Total number of assets.

Expand All @@ -696,7 +701,9 @@ Finds assets satisfying provided criteria.
},
"metadata": {
"bundleId": "0x85a427a3.....cd1d38ebbd",
"bundleTransactionHash": "0x21ab....1cdf8e55b37"
"bundleTransactionHash": "0x21ab....1cdf8e55b37",
"bundleUploadTimestamp": 1503424969,
"bundleProofBlock": 3221
}
}
],
Expand Down Expand Up @@ -901,7 +908,8 @@ Fetches an event by identifier
+ bundleId (string) - If the event has already been added to a bundle, Id of the bundle holding the event
+ bundleTransactionHash (string) - If the event has already been added to a bundle, identifier of the transaction on which the proof of bundle has been uploaded.
More info on the transaction can be found under `http://explorer-test.ambrosus.com/#/tx/{bundleTransactionHash}`

+ bundleProofBlock (number) - If the event has already been added to a bundle, block number in which the proof of the bundle has been included
+ bundleUploadTimestamp (number) - If the event has already been added to a bundle, timestamp of bundle proof upload

+ Body

Expand Down Expand Up @@ -1040,7 +1048,6 @@ Fetches a bundle by identifier
+ entriesHash (string) - Hash calculated from the entries field.
+ entries (array) - An array consisting of assets, events with public data and stubs of events with private data.


+ Body

{
Expand Down Expand Up @@ -1083,16 +1090,16 @@ Fetches a bundle metadata by identifier
+ bundleId (string) - Content-addressable identifier of the bundle.
+ bundleTransactionHash (string) - Identifier of the transaction on which the proof of bundle has been uploaded.
More info on the transaction can be found under `http://explorer-test.ambrosus.com/#/tx/{bundleTransactionHash}`
+ proofBlock (number) - Block number in which the transaction was included
+ bundleUploadTimestamp (number) - Timestamp of bundle upload
+ bundleProofBlock (number) - Block number in which the proof of the bundle has been included
+ bundleUploadTimestamp (number) - Timestamp of bundle proof upload


+ Body

{
"bundleId" : "0xa9bfd34.....30795ed11",
"bundleTransactionHash": "0xc9087b7510e98183f705fe99ddb6964f3b845878d8a801cf6b110975599b6009",
"proofBlock": 142,
"bundleProofBlock": 142,
"bundleUploadTimestamp": 1503424969
}

Expand Down
1 change: 1 addition & 0 deletions dev.env
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ export DEFAULT_STAKE="10000000000000000000000"
export SENTRY_DSN=""
export SENTRY_IGNORE_ERRORS_REGEX='^(NotFoundError|ValidationError|JsonValidationError|PermissionError)'
export LOW_FUNDS_WARNING=1000
export WORKER_INTERVAL=60
12 changes: 9 additions & 3 deletions src/builder.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ import {
} from 'ambrosus-node-contracts';
import DataModelEngine from './services/data_model_engine';
import EntityBuilder from './services/entity_builder';
import EntityDownloader from './services/entity_downloader';
import EntityRepository from './services/entity_repository';
import BundleDownloader from './services/bundle_downloader';
import BundleBuilder from './services/bundle_builder';
import BundleRepository from './services/bundle_repository';
import WorkerLogRepository from './services/worker_log_repository';
import FindEventQueryObjectFactory from './services/find_event_query_object';
import FindAccountQueryObjectFactory from './services/find_account_query_object';
Expand Down Expand Up @@ -98,13 +100,15 @@ class Builder {
const {maximumEntityTimestampOvertake} = this.config;
this.entityBuilder = new EntityBuilder(this.identityManager, maximumEntityTimestampOvertake);
this.entityRepository = new EntityRepository(this.db);
this.bundleBuilder = new BundleBuilder(this.identityManager, this.entityBuilder);
this.bundleRepository = new BundleRepository(this.db);
this.workerLogRepository = new WorkerLogRepository(this.db);
this.workerTaskTrackingRepository = new WorkerTaskTrackingRepository(this.db);
this.findEventQueryObjectFactory = new FindEventQueryObjectFactory(this.db);
this.findAssetQueryObjectFactory = new FindAssetQueryObjectFactory(this.db);
this.failedChallengesCache = new FailedChallengesCache();
this.httpsClient = new HttpsClient();
this.entityDownloader = new EntityDownloader(this.httpsClient);
this.bundleDownloader = new BundleDownloader(this.httpsClient);
this.accountRepository = new AccountRepository(this.db);
this.findAccountQueryObjectFactory = new FindAccountQueryObjectFactory(this.db);
this.accountAccessDefinitions = new AccountAccessDefinitions(this.identityManager, this.accountRepository);
Expand All @@ -113,7 +117,9 @@ class Builder {
tokenAuthenticator: this.tokenAuthenticator,
entityBuilder: this.entityBuilder,
entityRepository: this.entityRepository,
entityDownloader: this.entityDownloader,
bundleDownloader: this.bundleDownloader,
bundleBuilder: this.bundleBuilder,
bundleRepository: this.bundleRepository,
accountRepository: this.accountRepository,
findEventQueryObjectFactory: this.findEventQueryObjectFactory,
findAccountQueryObjectFactory: this.findAccountQueryObjectFactory,
Expand Down
15 changes: 15 additions & 0 deletions src/migrations/20181212100000_proof_block_rename.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/*
Copyright: Ambrosus Technologies GmbH
Email: tech@ambrosus.com
This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0.
*/

// eslint-disable-next-line import/prefer-default-export
export const up = async (db, config, logger) => {
await db.collection('bundle_metadata').updateMany({}, {$rename: {proofBlock: 'bundleProofBlock'}});

logger.info(`Renamed proofBlock to bundleProofBlock in bundle metadata`);
};
76 changes: 76 additions & 0 deletions src/services/bundle_builder.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
Copyright: Ambrosus Technologies GmbH
Email: tech@ambrosus.com
This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0.
*/

import validateAndCast from '../utils/validations';

export default class BundleBuilder {
constructor(identityManager, entityBuilder) {
this.identityManager = identityManager;
this.entityBuilder = entityBuilder;
}

assembleBundle(assets, events, timestamp, secret) {
const createdBy = this.identityManager.addressFromSecret(secret);
const preparedEvents = events.map((event) => this.entityBuilder.prepareEventForBundlePublication(event));
const entries = [
...assets,
...preparedEvents
].map((entry) => this.entityBuilder.removeBundle(entry));
const entriesHash = this.identityManager.calculateHash(entries);
const idData = {
createdBy,
entriesHash,
timestamp
};
const signature = this.identityManager.sign(secret, idData);
const content = {
signature,
idData,
entries
};
const bundleId = this.identityManager.calculateHash(content);

return {
bundleId,
content
};
}

validateBundle(bundle) {
validateAndCast(bundle)
.required([
'bundleId',
'content.signature',
'content.idData',
'content.idData.createdBy',
'content.idData.timestamp',
'content.idData.entriesHash',
'content.entries'
])
.fieldsConstrainedToSet(['content', 'bundleId', 'metadata'])
.fieldsConstrainedToSet(['idData', 'entries', 'signature'], 'content')
.isNonNegativeInteger(['content.idData.timestamp'])
.validate(
['bundleId'],
(hash) => this.identityManager.checkHashMatches(hash, bundle.content),
`bundleId value doesn't match the content hash`
)
.validate(
['content.idData.entriesHash'],
(hash) => this.identityManager.checkHashMatches(hash, bundle.content.entries),
`entriesHash value doesn't match the entries hash`
);

this.identityManager.validateSignature(
bundle.content.idData.createdBy,
bundle.content.signature,
bundle.content.idData
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ This Source Code Form is subject to the terms of the Mozilla Public License, v.
This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0.
*/

export default class EntityDownloader {
export default class BundleDownloader {
constructor(httpsClient) {
this.httpsClient = httpsClient;
}
Expand Down
59 changes: 59 additions & 0 deletions src/services/bundle_repository.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
Copyright: Ambrosus Technologies GmbH
Email: tech@ambrosus.com
This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0.
*/

export default class BundleRepository {
constructor(db) {
this.db = db;
this.blacklistedFields = {
_id: 0,
repository: 0
};
}

async storeBundle(bundle, storagePeriods) {
if (await this.db.collection('bundles').findOne({bundleId: bundle.bundleId}) === null) {
await this.db.collection('bundles').insertOne({...bundle});
}
if (await this.db.collection('bundle_metadata').findOne({bundleId: bundle.bundleId}) === null) {
await this.db.collection('bundle_metadata').insertOne({bundleId: bundle.bundleId, storagePeriods});
}
}

async storeBundleProofMetadata(bundleId, proofBlock, timestamp, txHash) {
await this.db.collection('bundle_metadata').updateOne({bundleId}, {
$set: {
bundleTransactionHash: txHash,
bundleProofBlock: proofBlock,
bundleUploadTimestamp: timestamp
}
});
}

async findBundlesWaitingForUpload() {
return await this.db.collection('bundle_metadata')
.find({bundleProofBlock: {$exists: false}})
.toArray();
}

async storeBundleShelteringExpirationDate(bundleId, expirationDate) {
await this.db.collection('bundles').updateOne({bundleId}, {
$set: {
'repository.holdUntil': expirationDate
}
});
}

async getBundle(bundleId) {
return await this.db.collection('bundles').findOne({bundleId}, {projection: this.blacklistedFields});
}

async getBundleMetadata(bundleId) {
return await this.db.collection('bundle_metadata').findOne({bundleId}, {projection: this.blacklistedFields});
}
}
28 changes: 16 additions & 12 deletions src/services/data_model_engine.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ import {pick, put} from '../utils/dict_utils';
import allPermissions from '../utils/all_permissions';

export default class DataModelEngine {
constructor({identityManager, tokenAuthenticator, entityBuilder, entityRepository, entityDownloader, accountRepository, findEventQueryObjectFactory, findAccountQueryObjectFactory, findAssetQueryObjectFactory, accountAccessDefinitions, mongoClient, uploadRepository, rolesRepository, workerLogRepository}) {
constructor({identityManager, tokenAuthenticator, entityBuilder, entityRepository, bundleDownloader, bundleBuilder, bundleRepository, accountRepository, findEventQueryObjectFactory, findAccountQueryObjectFactory, findAssetQueryObjectFactory, accountAccessDefinitions, mongoClient, uploadRepository, rolesRepository, workerLogRepository}) {
this.identityManager = identityManager;
this.tokenAuthenticator = tokenAuthenticator;
this.entityBuilder = entityBuilder;
this.entityRepository = entityRepository;
this.entityDownloader = entityDownloader;
this.bundleDownloader = bundleDownloader;
this.bundleBuilder = bundleBuilder;
this.bundleRepository = bundleRepository;
this.accountRepository = accountRepository;
this.findEventQueryObjectFactory = findEventQueryObjectFactory;
this.findAccountQueryObjectFactory = findAccountQueryObjectFactory;
Expand Down Expand Up @@ -169,15 +171,15 @@ export default class DataModelEngine {
}

async getBundle(bundleId) {
const bundle = await this.entityRepository.getBundle(bundleId);
const bundle = await this.bundleRepository.getBundle(bundleId);
if (bundle === null) {
throw new NotFoundError(`No bundle with id = ${bundleId} found`);
}
return bundle;
}

async getBundleMetadata(bundleId) {
const metadata = await this.entityRepository.getBundleMetadata(bundleId);
const metadata = await this.bundleRepository.getBundleMetadata(bundleId);
if (metadata === null) {
throw new NotFoundError(`No metadata found for bundleId = ${bundleId}`);
}
Expand All @@ -189,11 +191,11 @@ export default class DataModelEngine {
const notBundled = await this.entityRepository.fetchEntitiesForBundling(bundleStubId, bundleItemsCountLimit);

const nodeSecret = await this.identityManager.nodePrivateKey();
return this.entityBuilder.assembleBundle(notBundled.assets, notBundled.events, getTimestamp(), nodeSecret);
return this.bundleBuilder.assembleBundle(notBundled.assets, notBundled.events, getTimestamp(), nodeSecret);
}

async acceptBundleCandidate(newBundle, bundleStubId, storagePeriods) {
await this.entityRepository.storeBundle(newBundle, storagePeriods);
await this.bundleRepository.storeBundle(newBundle, storagePeriods);
await this.entityRepository.markEntitiesAsBundled(bundleStubId, newBundle.bundleId);

return newBundle;
Expand All @@ -204,15 +206,17 @@ export default class DataModelEngine {
}

async uploadAcceptedBundleCandidates() {
const waitingBundles = await this.entityRepository.findBundlesWaitingForUpload();
const waitingBundles = await this.bundleRepository.findBundlesWaitingForUpload();
const summary = {
ok: {},
failed: {}
};
for (const waitingBundle of waitingBundles) {
try {
const {blockNumber, transactionHash, uploadResult} = await this.uploadRepository.ensureBundleIsUploaded(waitingBundle.bundleId, waitingBundle.storagePeriods);
await this.entityRepository.storeBundleProofMetadata(waitingBundle.bundleId, blockNumber, transactionHash);
const timestamp = getTimestamp();
await this.entityRepository.storeBundleProofMetadata(waitingBundle.bundleId, blockNumber, timestamp, transactionHash);
await this.bundleRepository.storeBundleProofMetadata(waitingBundle.bundleId, blockNumber, timestamp, transactionHash);
summary.ok[waitingBundle.bundleId] = {uploadResult};
} catch (err) {
summary.failed[waitingBundle.bundleId] = err;
Expand All @@ -223,19 +227,19 @@ export default class DataModelEngine {

async downloadBundle(bundleId, sheltererId) {
const nodeUrl = await this.rolesRepository.nodeUrl(sheltererId);
const bundle = await this.entityDownloader.downloadBundle(nodeUrl, bundleId);
const bundle = await this.bundleDownloader.downloadBundle(nodeUrl, bundleId);
if (!bundle) {
throw new Error('Could not fetch the bundle from the shelterer');
}
this.entityBuilder.validateBundle(bundle);
this.bundleBuilder.validateBundle(bundle);
await this.uploadRepository.verifyBundle(bundle);
await this.entityRepository.storeBundle(bundle);
await this.bundleRepository.storeBundle(bundle);
return bundle;
}

async updateShelteringExpirationDate(bundleId) {
const expirationDate = await this.uploadRepository.expirationDate(bundleId);
await this.entityRepository.storeBundleShelteringExpirationDate(bundleId, expirationDate);
await this.bundleRepository.storeBundleShelteringExpirationDate(bundleId, expirationDate);
}

async getWorkerLogs(logsCount = 10) {
Expand Down
Loading

0 comments on commit 06a1da0

Please sign in to comment.