From b10fe81b2049a0dae61337e6424a2b2390287b9a Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 30 Mar 2021 12:22:13 +0300 Subject: [PATCH 01/31] ETH-81: Expose transportSignatures for an alternative withdraw flow --- src/dataunion/Contracts.ts | 41 +---------- src/dataunion/DataUnion.ts | 137 +++++++++++++++++++++++++++++-------- 2 files changed, 108 insertions(+), 70 deletions(-) diff --git a/src/dataunion/Contracts.ts b/src/dataunion/Contracts.ts index ebb97dc8c..a5d5ff6b4 100644 --- a/src/dataunion/Contracts.ts +++ b/src/dataunion/Contracts.ts @@ -171,7 +171,7 @@ export class Contracts { const alreadyProcessed = await mainnetAmb.relayedMessages(messageId) if (alreadyProcessed) { log(`WARNING: Tried to transport signatures but they have already been transported (Message ${messageId} has already been processed)`) - log('This could happen if freeWithdraw=false (attempt self-service), but bridge actually paid before your client') + log('This could happen if bridge paid for transport before your client.') return null } @@ -218,45 +218,6 @@ export class Contracts { return trAMB } - async transportSignaturesForTransaction(tr: ContractReceipt, options: { pollingIntervalMs?: number, retryTimeoutMs?: number } = {}) { - const { - pollingIntervalMs = 1000, - retryTimeoutMs = 60000, - } = options - log(`Got receipt, filtering UserRequestForSignature from ${tr.events!.length} events...`) - // event UserRequestForSignature(bytes32 indexed messageId, bytes encodedData); - const sigEventArgsArray = tr.events!.filter((e: Todo) => e.event === 'UserRequestForSignature').map((e: Todo) => e.args) - if (sigEventArgsArray.length < 1) { - throw new Error("No UserRequestForSignature events emitted from withdraw transaction, can't transport withdraw to mainnet") - } - - /* eslint-disable no-await-in-loop */ - // eslint-disable-next-line no-restricted-syntax - for (const eventArgs of sigEventArgsArray) { - const messageId = eventArgs[0] - const messageHash = keccak256(eventArgs[1]) - - log(`Waiting until sidechain AMB has collected required signatures for hash=${messageHash}...`) - await until(async () => this.requiredSignaturesHaveBeenCollected(messageHash), pollingIntervalMs, retryTimeoutMs) - - log(`Checking mainnet AMB hasn't already processed messageId=${messageId}`) - const mainnetAmb = await this.getMainnetAmb() - const alreadySent = await mainnetAmb.messageCallStatus(messageId) - const failAddress = await mainnetAmb.failedMessageSender(messageId) - - // zero address means no failed messages - if (alreadySent || failAddress !== '0x0000000000000000000000000000000000000000') { - log(`WARNING: Mainnet bridge has already processed withdraw messageId=${messageId}`) - log('This could happen if freeWithdraw=false (attempt self-service), but bridge actually paid before your client') - continue - } - - log(`Transporting signatures for hash=${messageHash}`) - await this.transportSignaturesForMessage(messageHash) - } - /* eslint-enable no-await-in-loop */ - } - async deployDataUnion({ ownerAddress, agentAddressList, diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index a2ad23c59..8a4d49214 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -1,16 +1,19 @@ import { getAddress } from '@ethersproject/address' import { BigNumber } from '@ethersproject/bignumber' import { arrayify, hexZeroPad } from '@ethersproject/bytes' -import { Contract } from '@ethersproject/contracts' +import { Contract, ContractReceipt, ContractTransaction } from '@ethersproject/contracts' +import { keccak256 } from '@ethersproject/keccak256' import { Wallet } from '@ethersproject/wallet' -import { JsonRpcSigner, TransactionReceipt, TransactionResponse } from '@ethersproject/providers' +import { JsonRpcSigner } from '@ethersproject/providers' import debug from 'debug' -import { Contracts } from './Contracts' + import { StreamrClient } from '../StreamrClient' import { EthereumAddress } from '../types' import { until, getEndpointUrl } from '../utils' import authFetch from '../rest/authFetch' +import { Contracts } from './Contracts' + export interface DataUnionDeployOptions { owner?: EthereumAddress, joinPartAgents?: EthereumAddress[], @@ -36,7 +39,8 @@ export interface JoinResponse { export interface DataUnionWithdrawOptions { pollingIntervalMs?: number retryTimeoutMs?: number - freeWithdraw?: boolean + transportSignatures?: boolean + waitUntilTransportIsComplete?: boolean sendToMainnet?: boolean } @@ -62,8 +66,17 @@ export interface MemberStats { withdrawableEarnings: BigNumber } +export type AmbMessageHash = string + const log = debug('StreamrClient::DataUnion') +function getMessageHashes(tr: ContractReceipt): AmbMessageHash[] { + // event UserRequestForSignature(bytes32 indexed messageId, bytes encodedData); + const sigEventArgsArray = tr.events!.filter((e) => e.event === 'UserRequestForSignature').map((e) => e.args) + const hashes = sigEventArgsArray.map((eventArgs) => keccak256(eventArgs![1])) + return hashes +} + /** * @category Important */ @@ -130,9 +143,9 @@ export class DataUnion { /** * Withdraw all your earnings - * @returns receipt once withdraw is complete (tokens are seen in mainnet) + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false */ - async withdrawAll(options?: DataUnionWithdrawOptions): Promise { + async withdrawAll(options?: DataUnionWithdrawOptions) { const recipientAddress = await this.client.getAddress() return this._executeWithdraw( () => this.getWithdrawAllTx(options?.sendToMainnet), @@ -145,7 +158,7 @@ export class DataUnion { * Get the tx promise for withdrawing all your earnings * @returns await on call .wait to actually send the tx */ - private async getWithdrawAllTx(sendToMainnet: boolean = true): Promise { + private async getWithdrawAllTx(sendToMainnet: boolean = true): Promise { const signer = await this.client.ethereum.getSidechainSigner() const address = await signer.getAddress() const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) @@ -164,12 +177,12 @@ export class DataUnion { /** * Withdraw earnings and "donate" them to the given address - * @returns get receipt once withdraw is complete (tokens are seen in mainnet) + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false */ async withdrawAllTo( recipientAddress: EthereumAddress, options?: DataUnionWithdrawOptions - ): Promise { + ) { const to = getAddress(recipientAddress) // throws if bad address return this._executeWithdraw( () => this.getWithdrawAllToTx(to, options?.sendToMainnet), @@ -183,7 +196,7 @@ export class DataUnion { * @param recipientAddress - the address to receive the tokens * @returns await on call .wait to actually send the tx */ - private async getWithdrawAllToTx(recipientAddress: EthereumAddress, sendToMainnet: boolean = true): Promise { + private async getWithdrawAllToTx(recipientAddress: EthereumAddress, sendToMainnet: boolean = true): Promise { const signer = await this.client.ethereum.getSidechainSigner() const address = await signer.getAddress() const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) @@ -365,7 +378,7 @@ export class DataUnion { */ async addMembers( memberAddressList: EthereumAddress[], - ): Promise { + ) { const members = memberAddressList.map(getAddress) // throws if there are bad addresses const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) const tx = await duSidechain.addMembers(members) @@ -378,7 +391,7 @@ export class DataUnion { */ async removeMembers( memberAddressList: EthereumAddress[], - ): Promise { + ) { const members = memberAddressList.map(getAddress) // throws if there are bad addresses const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) const tx = await duSidechain.partMembers(members) @@ -390,12 +403,12 @@ export class DataUnion { * Admin: withdraw earnings (pay gas) on behalf of a member * TODO: add test * @param memberAddress - the other member who gets their tokens out of the Data Union - * @returns Receipt once withdraw transaction is confirmed + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false */ async withdrawAllToMember( memberAddress: EthereumAddress, options?: DataUnionWithdrawOptions - ): Promise { + ) { const address = getAddress(memberAddress) // throws if bad address return this._executeWithdraw( () => this.getWithdrawAllToMemberTx(address, options?.sendToMainnet), @@ -409,7 +422,7 @@ export class DataUnion { * @param memberAddress - the other member who gets their tokens out of the Data Union * @returns await on call .wait to actually send the tx */ - private async getWithdrawAllToMemberTx(memberAddress: EthereumAddress, sendToMainnet: boolean = true): Promise { + private async getWithdrawAllToMemberTx(memberAddress: EthereumAddress, sendToMainnet: boolean = true): Promise { const a = getAddress(memberAddress) // throws if bad address const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) return duSidechain.withdrawAll(a, sendToMainnet) @@ -420,14 +433,14 @@ export class DataUnion { * @param memberAddress - the member whose earnings are sent out * @param recipientAddress - the address to receive the tokens in mainnet * @param signature - from member, produced using signWithdrawAllTo - * @returns receipt once withdraw transaction is confirmed + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false */ async withdrawAllToSigned( memberAddress: EthereumAddress, recipientAddress: EthereumAddress, signature: string, options?: DataUnionWithdrawOptions - ): Promise { + ) { const from = getAddress(memberAddress) // throws if bad address const to = getAddress(recipientAddress) return this._executeWithdraw( @@ -449,7 +462,7 @@ export class DataUnion { recipientAddress: EthereumAddress, signature: string, sendToMainnet: boolean = true, - ): Promise { + ) { const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) return duSidechain.withdrawAllToSigned(memberAddress, recipientAddress, sendToMainnet, signature) } @@ -457,7 +470,7 @@ export class DataUnion { /** * Admin: set admin fee (between 0.0 and 1.0) for the data union */ - async setAdminFee(newFeeFraction: number): Promise { + async setAdminFee(newFeeFraction: number) { if (newFeeFraction < 0 || newFeeFraction > 1) { throw new Error('newFeeFraction argument must be a number between 0...1, got: ' + newFeeFraction) } @@ -549,30 +562,94 @@ export class DataUnion { return new Contracts(this.client) } - // template for withdraw functions - // client could be replaced with AMB (mainnet and sidechain) + /** + * Template for withdraw functions + * @private + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false + */ private async _executeWithdraw( - getWithdrawTxFunc: () => Promise, + getWithdrawTxFunc: () => Promise, recipientAddress: EthereumAddress, options: DataUnionWithdrawOptions = {} - ): Promise { + ): Promise { const { pollingIntervalMs = 1000, retryTimeoutMs = 60000, - freeWithdraw = this.client.options.dataUnion.freeWithdraw, + transportSignatures = !this.client.options.dataUnion.freeWithdraw, // by default, transport the signatures if freeWithdraw isn't supported by the sidechain + waitUntilTransportIsComplete = true, sendToMainnet = true, } = options + const getBalanceFunc = sendToMainnet ? () => this.client.getTokenBalance(recipientAddress) : () => this.client.getSidechainTokenBalance(recipientAddress) - const balanceBefore = await getBalanceFunc() + const balanceBefore = waitUntilTransportIsComplete ? await getBalanceFunc() : 0 + + log('Executing DataUnionSidechain withdraw function') const tx = await getWithdrawTxFunc() const tr = await tx.wait() - if (!freeWithdraw && sendToMainnet) { - await this.getContracts().transportSignaturesForTransaction(tr, options) + + // keep tokens in the sidechain => just return the sidechain tx receipt + if (!sendToMainnet) { return tr } + + log(`Got receipt, filtering UserRequestForSignature from ${tr.events!.length} events...`) + const ambHashes = getMessageHashes(tr) + + if (ambHashes.length < 1) { throw new Error("No UserRequestForSignature events emitted from withdraw transaction, can't transport withdraw to mainnet") } + + if (ambHashes.length > 1) { throw new Error('Expected only one UserRequestForSignature event') } + + const messageHash = ambHashes[0] + + if (!transportSignatures) { + // expect someone else to do the transport for us (corresponds to dataUnion.freeWithdraw=true) + if (waitUntilTransportIsComplete) { + log(`Waiting for balance ${balanceBefore.toString()} to change`) + await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs) + return null + } + + // instead of waiting, hand out the messageHash so that someone else might do the transport using it + return messageHash + } + + const ambTr = await this.transportMessage(messageHash, pollingIntervalMs, retryTimeoutMs) + if (waitUntilTransportIsComplete) { + log(`Waiting for balance ${balanceBefore.toString()} to change`) + await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs) + } + return ambTr + } + + /** + * @returns null if message was already transported, ELSE the mainnet AMB signature execution transaction receipt + */ + async transportMessage(messageHash: AmbMessageHash, pollingIntervalMs?: number, retryTimeoutMs?: number) { + const helper = this.getContracts() + const sidechainAmb = await helper.getSidechainAmb() + const mainnetAmb = await helper.getMainnetAmb() + const message = await sidechainAmb.message(messageHash) + + if (message === '0x') { + throw new Error(`Message with hash=${messageHash} not found`) } - log(`Waiting for balance ${balanceBefore.toString()} to change`) - await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs) - return tr + const messageId = '0x' + message.substr(2, 64) + + log(`Waiting until sidechain AMB has collected required signatures for hash=${messageHash}...`) + await until(async () => helper.requiredSignaturesHaveBeenCollected(messageHash), pollingIntervalMs, retryTimeoutMs) + + log(`Checking mainnet AMB hasn't already processed messageId=${messageId}`) + const alreadySent = await mainnetAmb.messageCallStatus(messageId) + const failAddress = await mainnetAmb.failedMessageSender(messageId) + + // zero address means no failed messages + if (alreadySent || failAddress !== '0x0000000000000000000000000000000000000000') { + log(`WARNING: Tried to transport signatures but they have already been transported (Message ${messageId} has already been processed)`) + log('This could happen if bridge paid for transport before your client.') + return null + } + + log(`Transporting signatures for hash=${messageHash}`) + return helper.transportSignaturesForMessage(messageHash) } } From 99f2abf4e2126e46a5bb0da1c28abc6c337e20bc Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 30 Mar 2021 12:36:21 +0300 Subject: [PATCH 02/31] make eslint happy --- src/dataunion/Contracts.ts | 2 +- src/dataunion/DataUnion.ts | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/dataunion/Contracts.ts b/src/dataunion/Contracts.ts index a5d5ff6b4..0d00177bf 100644 --- a/src/dataunion/Contracts.ts +++ b/src/dataunion/Contracts.ts @@ -1,6 +1,6 @@ import { getCreate2Address, isAddress } from '@ethersproject/address' import { arrayify, hexZeroPad } from '@ethersproject/bytes' -import { Contract, ContractReceipt } from '@ethersproject/contracts' +import { Contract } from '@ethersproject/contracts' import { keccak256 } from '@ethersproject/keccak256' import { defaultAbiCoder } from '@ethersproject/abi' import { verifyMessage } from '@ethersproject/wallet' diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index 8a4d49214..c5b442fc9 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -143,7 +143,10 @@ export class DataUnion { /** * Withdraw all your earnings - * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, + * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, + * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ async withdrawAll(options?: DataUnionWithdrawOptions) { const recipientAddress = await this.client.getAddress() @@ -177,7 +180,10 @@ export class DataUnion { /** * Withdraw earnings and "donate" them to the given address - * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, + * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, + * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ async withdrawAllTo( recipientAddress: EthereumAddress, @@ -403,7 +409,10 @@ export class DataUnion { * Admin: withdraw earnings (pay gas) on behalf of a member * TODO: add test * @param memberAddress - the other member who gets their tokens out of the Data Union - * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, + * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, + * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ async withdrawAllToMember( memberAddress: EthereumAddress, @@ -433,7 +442,10 @@ export class DataUnion { * @param memberAddress - the member whose earnings are sent out * @param recipientAddress - the address to receive the tokens in mainnet * @param signature - from member, produced using signWithdrawAllTo - * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, + * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, + * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ async withdrawAllToSigned( memberAddress: EthereumAddress, @@ -565,7 +577,10 @@ export class DataUnion { /** * Template for withdraw functions * @private - * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost), ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false + * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, + * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, + * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ private async _executeWithdraw( getWithdrawTxFunc: () => Promise, @@ -575,7 +590,8 @@ export class DataUnion { const { pollingIntervalMs = 1000, retryTimeoutMs = 60000, - transportSignatures = !this.client.options.dataUnion.freeWithdraw, // by default, transport the signatures if freeWithdraw isn't supported by the sidechain + // by default, transport the signatures if freeWithdraw isn't supported by the sidechain + transportSignatures = !this.client.options.dataUnion.freeWithdraw, waitUntilTransportIsComplete = true, sendToMainnet = true, } = options @@ -595,9 +611,13 @@ export class DataUnion { log(`Got receipt, filtering UserRequestForSignature from ${tr.events!.length} events...`) const ambHashes = getMessageHashes(tr) - if (ambHashes.length < 1) { throw new Error("No UserRequestForSignature events emitted from withdraw transaction, can't transport withdraw to mainnet") } + if (ambHashes.length < 1) { + throw new Error("No UserRequestForSignature events emitted from withdraw transaction, can't transport withdraw to mainnet") + } - if (ambHashes.length > 1) { throw new Error('Expected only one UserRequestForSignature event') } + if (ambHashes.length > 1) { + throw new Error('Expected only one UserRequestForSignature event') + } const messageHash = ambHashes[0] From 5e0a208d413a30d05d1b6532e211e58b57c3e4e5 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 30 Mar 2021 15:46:28 +0300 Subject: [PATCH 03/31] Update README with two-(three?)step withdraw --- README.md | 60 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e103ac5a7..9d9232058 100644 --- a/README.md +++ b/README.md @@ -375,8 +375,9 @@ These DataUnion-specific options can be given to `new StreamrClient` options: | setAdminFee(newFeeFraction) | Transaction receipt | `newFeeFraction` is a `Number` between 0.0 and 1.0 (inclusive) | | addMembers(memberAddressList) | Transaction receipt | Add members | | removeMembers(memberAddressList) | Transaction receipt | Remove members from Data Union | -| withdrawAllToMember(memberAddress\[, [options](#withdraw-options)\]) | Transaction receipt | Send all withdrawable earnings to the member's address | -| withdrawAllToSigned(memberAddress, recipientAddress, signature\[, [options](#withdraw-options)\]) | Transaction receipt | Send all withdrawable earnings to the address signed off by the member (see [example below](#member-functions)) | +| withdrawAllToMember(memberAddress\[, [options](#withdraw-options)\]) | Transaction receipt `*` | Send all withdrawable earnings to the member's address | +| withdrawAllToSigned(memberAddress, recipientAddress, signature\[, [options](#withdraw-options)\]) | Transaction receipt `*` | Send all withdrawable earnings to the address signed off by the member (see [example below](#member-functions)) | +`*` The return value type may vary depending on [the given options](#withdraw-options) that describe the use case. Here's how to deploy a Data Union contract with 30% Admin fee and add some members: @@ -399,14 +400,16 @@ const receipt = await dataUnion.addMembers([ ### Member functions -| Name | Returns | Description | -| :---------------------------------------------------------------- | :------------------ | :-------------------------------------------------------------------------- | -| join(\[secret]) | JoinRequest | Join the Data Union (if a valid secret is given, the promise waits until the automatic join request has been processed) | -| isMember(memberAddress) | boolean | | -| withdrawAll(\[[options](#withdraw-options)\]) | Transaction receipt | Withdraw funds from Data Union | -| withdrawAllTo(recipientAddress\[, [options](#withdraw-options)\]) | Transaction receipt | Donate/move your earnings to recipientAddress instead of your memberAddress | -| signWithdrawAllTo(recipientAddress) | Signature (string) | Signature that can be used to withdraw all available tokens to given recipientAddress | -| signWithdrawAmountTo(recipientAddress, amountTokenWei) | Signature (string) | Signature that can be used to withdraw a specific amount of tokens to given recipientAddress | +| Name | Returns | Description | +| :-------------------------------------------------------------------- | :------------------------ | :-------------------------------------------------------------------------- | +| join(\[secret]) | JoinRequest | Join the Data Union (if a valid secret is given, the promise waits until the automatic join request has been processed) | +| isMember(memberAddress) | boolean | | +| withdrawAll(\[[options](#withdraw-options)\]) | Transaction receipt `*` | Withdraw funds from Data Union | +| withdrawAllTo(recipientAddress\[, [options](#withdraw-options)\]) | Transaction receipt `*` | Donate/move your earnings to recipientAddress instead of your memberAddress | +| signWithdrawAllTo(recipientAddress) | Signature (string) | Signature that can be used to withdraw all available tokens to given recipientAddress | +| signWithdrawAmountTo(recipientAddress, amountTokenWei) | Signature (string) | Signature that can be used to withdraw a specific amount of tokens to given recipientAddress | +| transportMessage(messageHash[, pollingIntervalMs[, retryTimeoutMs]]) | Transaction receipt | Send the mainnet transaction to withdraw tokens from the sidechain | +`*` The return value type may vary depending on [the given options](#withdraw-options) that describe the use case. Here's an example on how to sign off on a withdraw to (any) recipientAddress (NOTE: this requires no gas!) @@ -434,6 +437,15 @@ const dataUnion = client.getDataUnion(dataUnionAddress) const receipt = await dataUnion.withdrawAllToSigned(memberAddress, recipientAddress, signature) ``` +The `messageHash` argument to `transportMessage` will come from the withdraw function with the specific options. The following is equivalent to the above withdraw line: +```js +const messageHash = await dataUnion.withdrawAllToSigned(memberAddress, recipientAddress, signature, { + transportSignatures: false, + waitUntilTransportIsComplete: false, +}) // only pay for sidechain gas +const receipt = await dataUnion.transportMessage(messageHash) // only pay for mainnet gas +``` + ### Query functions These are available for everyone and anyone, to query publicly available info from a Data Union: @@ -460,13 +472,29 @@ const withdrawableWei = await dataUnion.getWithdrawableEarnings(memberAddress) The functions `withdrawAll`, `withdrawAllTo`, `withdrawAllToMember`, `withdrawAllToSigned` all can take an extra "options" argument. It's an object that can contain the following parameters: -| Name | Default | Description | -| :---------------- | :-------------------- | :---------------------------------------------------------------------------------- | -| sendToMainnet | true | Whether to send the withdrawn DATA tokens to mainnet address (or sidechain address) | -| pollingIntervalMs | 1000 (1 second) | How often requests are sent to find out if the withdraw has completed | -| retryTimeoutMs | 60000 (1 minute) | When to give up when waiting for the withdraw to complete | +| Name | Default | Description | +| :---------------- | :-------------------- | :---------------------------------------------------------------------------------- | +| sendToMainnet | true | Whether to send the withdrawn DATA tokens to mainnet address (or sidechain address) | +| transportSignatures | true | Whether to pay for the withdraw transaction signature transport to mainnet over the bridge | +| waitUntilTransportIsComplete | true | Whether to wait until the withdrawn DATA tokens are visible in mainnet | +| pollingIntervalMs | 1000 (1 second) | How often requests are sent to find out if the withdraw has completed | +| retryTimeoutMs | 60000 (1 minute) | When to give up when waiting for the withdraw to complete | + +These withdraw transactions are sent to the sidechain, so gas price shouldn't be manually set (fees will hopefully stay very low), +but a little bit of [sidechain native token](https://www.xdaichain.com/for-users/get-xdai-tokens) is nonetheless required. + +The return values from the withdraw functions also depend on the options. + +If `sendToMainnet: false`, other options don't apply at all, and **sidechain transaction receipt** is returned as soon as the withdraw transaction is done. This should be fairly quick in the sidechain. + +The use cases corresponding to the different combinations of the boolean flags: -These withdraw transactions are sent to the sidechain, so gas price shouldn't be manually set (fees will hopefully stay very low), but a little bit of [sidechain native token](https://www.xdaichain.com/for-users/get-xdai-tokens) is nonetheless required. +| `transport` | `wait` | Returns | Effect | +| :---------- | :------ | :------ | :----- | +| `true` | `true` | Transaction receipt | *(default)* Self-service bridge to mainnet, client pays for mainnet gas | +| `true` | `false` | Transaction receipt | Self-service bridge to mainnet (but **skip** the wait that double-checks the withdraw succeeded and tokens arrived to destination) | +| `false` | `true` | `null` | Someone else pays for the mainnet gas automatically, e.g. the bridge operator (in this case the transaction receipt can't be returned) | +| `false` | `false` | AMB message hash | Someone else pays for the mainnet gas, but we need to give them the message hash first | ### Deployment options From b52a2ec6ad1f1bb400dbf345a8a0fabc38415ef3 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Wed, 31 Mar 2021 15:38:02 +0300 Subject: [PATCH 04/31] fix tables --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d9232058..fef5b0351 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,7 @@ These DataUnion-specific options can be given to `new StreamrClient` options: | removeMembers(memberAddressList) | Transaction receipt | Remove members from Data Union | | withdrawAllToMember(memberAddress\[, [options](#withdraw-options)\]) | Transaction receipt `*` | Send all withdrawable earnings to the member's address | | withdrawAllToSigned(memberAddress, recipientAddress, signature\[, [options](#withdraw-options)\]) | Transaction receipt `*` | Send all withdrawable earnings to the address signed off by the member (see [example below](#member-functions)) | + `*` The return value type may vary depending on [the given options](#withdraw-options) that describe the use case. Here's how to deploy a Data Union contract with 30% Admin fee and add some members: @@ -408,7 +409,8 @@ const receipt = await dataUnion.addMembers([ | withdrawAllTo(recipientAddress\[, [options](#withdraw-options)\]) | Transaction receipt `*` | Donate/move your earnings to recipientAddress instead of your memberAddress | | signWithdrawAllTo(recipientAddress) | Signature (string) | Signature that can be used to withdraw all available tokens to given recipientAddress | | signWithdrawAmountTo(recipientAddress, amountTokenWei) | Signature (string) | Signature that can be used to withdraw a specific amount of tokens to given recipientAddress | -| transportMessage(messageHash[, pollingIntervalMs[, retryTimeoutMs]]) | Transaction receipt | Send the mainnet transaction to withdraw tokens from the sidechain | +| transportMessage(messageHash[, pollingIntervalMs[, retryTimeoutMs]]) | Transaction receipt | Send the mainnet transaction to withdraw tokens from the sidechain | + `*` The return value type may vary depending on [the given options](#withdraw-options) that describe the use case. Here's an example on how to sign off on a withdraw to (any) recipientAddress (NOTE: this requires no gas!) From 42d99e858bb292cdff7ea9a0cd73d85c4d4379e5 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Fri, 16 Apr 2021 15:18:34 +0300 Subject: [PATCH 05/31] transportSignature -> payForTransport because the method to do it is transportMessage and that's really all it means, so either it should be the SAME name, or else a bit more descriptive --- README.md | 4 ++-- src/dataunion/DataUnion.ts | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fef5b0351..f926d1616 100644 --- a/README.md +++ b/README.md @@ -442,7 +442,7 @@ const receipt = await dataUnion.withdrawAllToSigned(memberAddress, recipientAddr The `messageHash` argument to `transportMessage` will come from the withdraw function with the specific options. The following is equivalent to the above withdraw line: ```js const messageHash = await dataUnion.withdrawAllToSigned(memberAddress, recipientAddress, signature, { - transportSignatures: false, + payForTransport: false, waitUntilTransportIsComplete: false, }) // only pay for sidechain gas const receipt = await dataUnion.transportMessage(messageHash) // only pay for mainnet gas @@ -477,7 +477,7 @@ The functions `withdrawAll`, `withdrawAllTo`, `withdrawAllToMember`, `withdrawAl | Name | Default | Description | | :---------------- | :-------------------- | :---------------------------------------------------------------------------------- | | sendToMainnet | true | Whether to send the withdrawn DATA tokens to mainnet address (or sidechain address) | -| transportSignatures | true | Whether to pay for the withdraw transaction signature transport to mainnet over the bridge | +| payForTransport | true | Whether to pay for the withdraw transaction signature transport to mainnet over the bridge | | waitUntilTransportIsComplete | true | Whether to wait until the withdrawn DATA tokens are visible in mainnet | | pollingIntervalMs | 1000 (1 second) | How often requests are sent to find out if the withdraw has completed | | retryTimeoutMs | 60000 (1 minute) | When to give up when waiting for the withdraw to complete | diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index c5b442fc9..5ee62c3f0 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -39,7 +39,7 @@ export interface JoinResponse { export interface DataUnionWithdrawOptions { pollingIntervalMs?: number retryTimeoutMs?: number - transportSignatures?: boolean + payForTransport?: boolean waitUntilTransportIsComplete?: boolean sendToMainnet?: boolean } @@ -144,7 +144,7 @@ export class DataUnion { /** * Withdraw all your earnings * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, - * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the message hash IF called with payForTransport=false and waitUntilTransportIsComplete=false, * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ @@ -181,7 +181,7 @@ export class DataUnion { /** * Withdraw earnings and "donate" them to the given address * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, - * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the message hash IF called with payForTransport=false and waitUntilTransportIsComplete=false, * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ @@ -410,7 +410,7 @@ export class DataUnion { * TODO: add test * @param memberAddress - the other member who gets their tokens out of the Data Union * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, - * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the message hash IF called with payForTransport=false and waitUntilTransportIsComplete=false, * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ @@ -443,7 +443,7 @@ export class DataUnion { * @param recipientAddress - the address to receive the tokens in mainnet * @param signature - from member, produced using signWithdrawAllTo * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, - * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the message hash IF called with payForTransport=false and waitUntilTransportIsComplete=false, * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ @@ -578,7 +578,7 @@ export class DataUnion { * Template for withdraw functions * @private * @returns the sidechain withdraw transaction receipt IF called with sendToMainnet=false, - * ELSE the message hash IF called with transportSignatures=false and waitUntilTransportIsComplete=false, + * ELSE the message hash IF called with payForTransport=false and waitUntilTransportIsComplete=false, * ELSE the mainnet AMB signature execution transaction receipt IF we did the transport ourselves, * ELSE null IF transport to mainnet was done by someone else (in which case the receipt is lost) */ @@ -591,7 +591,7 @@ export class DataUnion { pollingIntervalMs = 1000, retryTimeoutMs = 60000, // by default, transport the signatures if freeWithdraw isn't supported by the sidechain - transportSignatures = !this.client.options.dataUnion.freeWithdraw, + payForTransport = !this.client.options.dataUnion.freeWithdraw, waitUntilTransportIsComplete = true, sendToMainnet = true, } = options @@ -621,7 +621,7 @@ export class DataUnion { const messageHash = ambHashes[0] - if (!transportSignatures) { + if (!payForTransport) { // expect someone else to do the transport for us (corresponds to dataUnion.freeWithdraw=true) if (waitUntilTransportIsComplete) { log(`Waiting for balance ${balanceBefore.toString()} to change`) From de6bbec277c400aa8661d9c355e3e32a575cc60a Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Fri, 16 Apr 2021 15:19:40 +0300 Subject: [PATCH 06/31] Added tests for each of the withdraw patterns documented in README also using describe.each instead of the homebrew loop. WebStorm recognizes this syntax, so please never use homebrew loops! --- test/integration/dataunion/withdraw.test.ts | 130 +++++++++++--------- 1 file changed, 72 insertions(+), 58 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 672b3fd55..807814ed6 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -1,6 +1,5 @@ import { BigNumber, Contract, providers, Wallet } from 'ethers' import { formatEther, parseEther } from 'ethers/lib/utils' -import { TransactionReceipt } from '@ethersproject/providers' import debug from 'debug' import { getEndpointUrl, until } from '../../../src/utils' @@ -10,7 +9,8 @@ import * as DataUnionSidechain from '../../../contracts/DataUnionSidechain.json' import config from '../config' import authFetch from '../../../src/rest/authFetch' import { createClient, createMockAddress, expectInvalidAddress } from '../../utils' -import { MemberStatus } from '../../../src/dataunion/DataUnion' +import { AmbMessageHash, DataUnionWithdrawOptions, MemberStatus } from '../../../src/dataunion/DataUnion' +import { ContractReceipt } from '@ethersproject/contracts' const log = debug('StreamrClient::DataUnion::integration-test-withdraw') @@ -31,8 +31,9 @@ const testWithdraw = async ( memberClient: StreamrClient, memberWallet: Wallet, adminClient: StreamrClient - ) => Promise, + ) => Promise, requiresMainnetETH: boolean, + options: DataUnionWithdrawOptions, ) => { log(`Connecting to Ethereum networks, config = ${JSON.stringify(config)}`) const network = await providerMainnet.getNetwork() @@ -135,8 +136,17 @@ const testWithdraw = async ( const balanceBefore = await getBalance(memberWallet) log(`Balance before: ${balanceBefore}. Withdrawing tokens...`) - const withdrawTr = await withdraw(dataUnion.getAddress(), memberClient, memberWallet, adminClient) - log(`Tokens withdrawn, sidechain tx receipt: ${JSON.stringify(withdrawTr)}`) + let ret = await withdraw(dataUnion.getAddress(), memberClient, memberWallet, adminClient) + if (ret instanceof String) { + log(`Transporting message "${ret}"`) + ret = await dataUnion.transportMessage(String(ret)) + } + log(`Tokens withdrawn, return value: ${JSON.stringify(ret)}`) + if (!options.waitUntilTransportIsComplete) { + log(`Waiting until balance changes from ${balanceBefore.toString()}`) + await until(async () => getBalance(memberWallet).then((b) => !b.eq(balanceBefore))) + } + const balanceAfter = await getBalance(memberWallet) const balanceIncrease = balanceAfter.sub(balanceBefore) @@ -158,65 +168,69 @@ describe('DataUnion withdraw', () => { providerSidechain.removeAllListeners() }) - for (const sendToMainnet of [true, false]) { + // TODO: add tests for just getting the hash and doing the transportMessage manually + describe.each([ + [false, true, true], // sidechain withdraw + [true, true, true], // self-service mainnet withdraw + [true, true, false], // self-service mainnet withdraw without checking the recipient account + [true, false, true], // bridge-sponsored mainnet withdraw + [true, false, false], // other-sponsored mainnet withdraw + ])('Withdrawing with sendToMainnet=%p, payForTransport=%p, wait=%p', (sendToMainnet, payForTransport, waitUntilTransportIsComplete) => { + const options = { sendToMainnet, payForTransport, waitUntilTransportIsComplete } const getTokenBalance = async (wallet: Wallet) => { return sendToMainnet ? balanceClient.getTokenBalance(wallet.address) : balanceClient.getSidechainTokenBalance(wallet.address) } - describe('Withdrawing to ' + (sendToMainnet ? 'mainnet' : 'sidechain'), () => { - - describe('Member', () => { - - it('by member itself', () => { - const getBalance = async (memberWallet: Wallet) => getTokenBalance(memberWallet) - const withdraw = async (dataUnionAddress: string, memberClient: StreamrClient) => ( - memberClient.getDataUnion(dataUnionAddress).withdrawAll({ sendToMainnet }) - ) - return testWithdraw(getBalance, withdraw, true) - }, 300000) - - it('from member to any address', () => { - const outsiderWallet = new Wallet(`0x100000000000000000000000000000000000000012300000002${Date.now()}`, providerSidechain) - const getBalance = async () => getTokenBalance(outsiderWallet) - const withdraw = (dataUnionAddress: string, memberClient: StreamrClient) => ( - memberClient.getDataUnion(dataUnionAddress).withdrawAllTo(outsiderWallet.address, { sendToMainnet }) - ) - return testWithdraw(getBalance, withdraw, true) - }, 300000) - - }) - - describe('Admin', () => { - - it('non-signed', async () => { - const getBalance = async (memberWallet: Wallet) => getTokenBalance(memberWallet) - const withdraw = (dataUnionAddress: string, _: StreamrClient, memberWallet: Wallet, adminClient: StreamrClient) => ( - adminClient.getDataUnion(dataUnionAddress).withdrawAllToMember(memberWallet.address, { sendToMainnet }) - ) - return testWithdraw(getBalance, withdraw, false) - }, 300000) - - it('signed', async () => { - const member2Wallet = new Wallet(`0x100000000000000000000000000040000000000012300000007${Date.now()}`, providerSidechain) - const getBalance = async () => getTokenBalance(member2Wallet) - const withdraw = async ( - dataUnionAddress: string, - memberClient: StreamrClient, - memberWallet: Wallet, - adminClient: StreamrClient - ) => { - const signature = await memberClient.getDataUnion(dataUnionAddress).signWithdrawAllTo(member2Wallet.address) - const withdrawTr = await adminClient - .getDataUnion(dataUnionAddress) - .withdrawAllToSigned(memberWallet.address, member2Wallet.address, signature, { sendToMainnet }) - return withdrawTr - } - return testWithdraw(getBalance, withdraw, false) - }, 300000) - }) + describe('by member', () => { + + it('to itself', () => { + const getBalance = async (memberWallet: Wallet) => getTokenBalance(memberWallet) + const withdraw = async (dataUnionAddress: string, memberClient: StreamrClient) => ( + memberClient.getDataUnion(dataUnionAddress).withdrawAll(options) + ) + return testWithdraw(getBalance, withdraw, true, options) + }, 300000) + + it('to any address', () => { + const outsiderWallet = new Wallet(`0x100000000000000000000000000000000000000012300000002${Date.now()}`, providerSidechain) + const getBalance = async () => getTokenBalance(outsiderWallet) + const withdraw = (dataUnionAddress: string, memberClient: StreamrClient) => ( + memberClient.getDataUnion(dataUnionAddress).withdrawAllTo(outsiderWallet.address, options) + ) + return testWithdraw(getBalance, withdraw, true, options) + }, 300000) + }) - } + + describe('by admin', () => { + + it('to member without signature', async () => { + const getBalance = async (memberWallet: Wallet) => getTokenBalance(memberWallet) + const withdraw = (dataUnionAddress: string, _: StreamrClient, memberWallet: Wallet, adminClient: StreamrClient) => ( + adminClient.getDataUnion(dataUnionAddress).withdrawAllToMember(memberWallet.address, options) + ) + return testWithdraw(getBalance, withdraw, false, options) + }, 300000) + + it("to anyone with member's signature", async () => { + const member2Wallet = new Wallet(`0x100000000000000000000000000040000000000012300000007${Date.now()}`, providerSidechain) + const getBalance = async () => getTokenBalance(member2Wallet) + const withdraw = async ( + dataUnionAddress: string, + memberClient: StreamrClient, + memberWallet: Wallet, + adminClient: StreamrClient + ) => { + const signature = await memberClient.getDataUnion(dataUnionAddress).signWithdrawAllTo(member2Wallet.address) + return adminClient + .getDataUnion(dataUnionAddress) + .withdrawAllToSigned(memberWallet.address, member2Wallet.address, signature, options) + } + return testWithdraw(getBalance, withdraw, false, options) + }, 300000) + }) + }) it('Validate address', async () => { const client = createClient(providerSidechain) From 501a15ba5abca5a1bec1f03997ecba81c13a2f41 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Mon, 19 Apr 2021 11:04:56 +0300 Subject: [PATCH 07/31] added type apparently ethers doesn't do enough to tell tx.wait() returns a ContractReceipt... --- src/dataunion/Contracts.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dataunion/Contracts.ts b/src/dataunion/Contracts.ts index 0d00177bf..e8b08920d 100644 --- a/src/dataunion/Contracts.ts +++ b/src/dataunion/Contracts.ts @@ -1,6 +1,6 @@ import { getCreate2Address, isAddress } from '@ethersproject/address' import { arrayify, hexZeroPad } from '@ethersproject/bytes' -import { Contract } from '@ethersproject/contracts' +import { Contract, ContractReceipt } from '@ethersproject/contracts' import { keccak256 } from '@ethersproject/keccak256' import { defaultAbiCoder } from '@ethersproject/abi' import { verifyMessage } from '@ethersproject/wallet' @@ -137,7 +137,7 @@ export class Contracts { } // move signatures from sidechain to mainnet - async transportSignaturesForMessage(messageHash: string) { + async transportSignaturesForMessage(messageHash: string): Promise { const sidechainAmb = await this.getSidechainAmb() const message = await sidechainAmb.message(messageHash) const messageId = '0x' + message.substr(2, 64) From 33b1715205306b65bfd65a6360a6c4fa91c07082 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Mon, 19 Apr 2021 16:16:57 +0300 Subject: [PATCH 08/31] fix timing issue --- src/dataunion/DataUnion.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index 5ee62c3f0..e22bc9a9c 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -648,16 +648,16 @@ export class DataUnion { const helper = this.getContracts() const sidechainAmb = await helper.getSidechainAmb() const mainnetAmb = await helper.getMainnetAmb() - const message = await sidechainAmb.message(messageHash) + log(`Waiting until sidechain AMB has collected required signatures for hash=${messageHash}...`) + await until(async () => helper.requiredSignaturesHaveBeenCollected(messageHash), pollingIntervalMs, retryTimeoutMs) + + const message = await sidechainAmb.message(messageHash) if (message === '0x') { throw new Error(`Message with hash=${messageHash} not found`) } const messageId = '0x' + message.substr(2, 64) - log(`Waiting until sidechain AMB has collected required signatures for hash=${messageHash}...`) - await until(async () => helper.requiredSignaturesHaveBeenCollected(messageHash), pollingIntervalMs, retryTimeoutMs) - log(`Checking mainnet AMB hasn't already processed messageId=${messageId}`) const alreadySent = await mainnetAmb.messageCallStatus(messageId) const failAddress = await mainnetAmb.failedMessageSender(messageId) From ad297a40683362056b780d343cebfc43cc59a04a Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Mon, 19 Apr 2021 16:18:01 +0300 Subject: [PATCH 09/31] added todo ticket number, cleanup --- src/dataunion/DataUnion.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index e22bc9a9c..b56a9f240 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -388,7 +388,7 @@ export class DataUnion { const members = memberAddressList.map(getAddress) // throws if there are bad addresses const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) const tx = await duSidechain.addMembers(members) - // TODO: wrap promise for better error reporting in case tx fails (parse reason, throw proper error) + // TODO ETH-93: wrap promise for better error reporting in case tx fails (parse reason, throw proper error) return tx.wait() } @@ -401,7 +401,7 @@ export class DataUnion { const members = memberAddressList.map(getAddress) // throws if there are bad addresses const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) const tx = await duSidechain.partMembers(members) - // TODO: wrap promise for better error reporting in case tx fails (parse reason, throw proper error) + // TODO ETH-93: wrap promise for better error reporting in case tx fails (parse reason, throw proper error) return tx.wait() } @@ -499,7 +499,6 @@ export class DataUnion { * @internal */ static async _deploy(options: DataUnionDeployOptions = {}, client: StreamrClient): Promise { - const deployerAddress = await client.getAddress() const { owner, joinPartAgents, @@ -510,6 +509,7 @@ export class DataUnion { confirmations = 1, gasPrice } = options + const deployerAddress = await client.getAddress() let duName = dataUnionName if (!duName) { From 02d4dbb1bcc0599c20de5ad0bb1855b35ec6c018 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 20 Apr 2021 16:43:01 +0300 Subject: [PATCH 10/31] fix broken type test strange that `instanceof String` doesn't actually catch a string --- test/integration/dataunion/withdraw.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 807814ed6..2d702906c 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -137,7 +137,7 @@ const testWithdraw = async ( log(`Balance before: ${balanceBefore}. Withdrawing tokens...`) let ret = await withdraw(dataUnion.getAddress(), memberClient, memberWallet, adminClient) - if (ret instanceof String) { + if (typeof ret === 'string') { log(`Transporting message "${ret}"`) ret = await dataUnion.transportMessage(String(ret)) } From 01840c349390f3bef1f328414e3520efbe202eb1 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Wed, 21 Apr 2021 15:10:24 +0300 Subject: [PATCH 11/31] Better error message --- src/dataunion/DataUnion.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index b56a9f240..1cdbea294 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -626,6 +626,7 @@ export class DataUnion { if (waitUntilTransportIsComplete) { log(`Waiting for balance ${balanceBefore.toString()} to change`) await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs) + .catch((e) => { throw e.message.startsWith('Timeout') ? new Error(`Timeout: Bridge did not transport withdraw message as expected. Fix: DataUnion.transportMessage("${messageHash}")`) : e }) return null } From 8ab79400de890e683349423fe5ba3bde31bb0ccc Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Thu, 22 Apr 2021 10:47:15 +0300 Subject: [PATCH 12/31] tweak timings for some reason it was polling every 100ms, which is definitely too much in production --- src/dataunion/DataUnion.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index 1cdbea294..a348d3a65 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -589,7 +589,7 @@ export class DataUnion { ): Promise { const { pollingIntervalMs = 1000, - retryTimeoutMs = 60000, + retryTimeoutMs = 300000, // by default, transport the signatures if freeWithdraw isn't supported by the sidechain payForTransport = !this.client.options.dataUnion.freeWithdraw, waitUntilTransportIsComplete = true, @@ -642,10 +642,14 @@ export class DataUnion { return ambTr } + // TODO: this doesn't belong here. Transporting a message is NOT dataunion-specific and needs nothing from DataUnion.ts. + // It shouldn't be required to create a DataUnion object to call this. + // This belongs to the StreamrClient, and if the code is too DU-specific then please shove it back to Contracts.ts. + // Division to transportMessage and Contracts.transportSignaturesForMessage is spurious, they should be one long function probably. /** * @returns null if message was already transported, ELSE the mainnet AMB signature execution transaction receipt */ - async transportMessage(messageHash: AmbMessageHash, pollingIntervalMs?: number, retryTimeoutMs?: number) { + async transportMessage(messageHash: AmbMessageHash, pollingIntervalMs: number = 1000, retryTimeoutMs: number = 300000) { const helper = this.getContracts() const sidechainAmb = await helper.getSidechainAmb() const mainnetAmb = await helper.getMainnetAmb() From 178a8e109342a34b9225594841751e7867578ae8 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Thu, 22 Apr 2021 10:48:29 +0300 Subject: [PATCH 13/31] fix the bridge-sponsored withdraw case tests should pass now (fingers crossed) --- test/integration/dataunion/withdraw.test.ts | 41 +++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 2d702906c..d7eaee397 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -1,5 +1,8 @@ import { BigNumber, Contract, providers, Wallet } from 'ethers' -import { formatEther, parseEther } from 'ethers/lib/utils' +import { formatEther, parseEther, defaultAbiCoder } from 'ethers/lib/utils' +import { ContractReceipt } from '@ethersproject/contracts' +import { keccak256 } from '@ethersproject/keccak256' + import debug from 'debug' import { getEndpointUrl, until } from '../../../src/utils' @@ -10,7 +13,6 @@ import config from '../config' import authFetch from '../../../src/rest/authFetch' import { createClient, createMockAddress, expectInvalidAddress } from '../../utils' import { AmbMessageHash, DataUnionWithdrawOptions, MemberStatus } from '../../../src/dataunion/DataUnion' -import { ContractReceipt } from '@ethersproject/contracts' const log = debug('StreamrClient::DataUnion::integration-test-withdraw') @@ -24,7 +26,7 @@ const tokenMainnet = new Contract(config.clientOptions.tokenAddress, Token.abi, const tokenSidechain = new Contract(config.clientOptions.tokenSidechainAddress, Token.abi, adminWalletSidechain) -const testWithdraw = async ( +async function testWithdraw( getBalance: (memberWallet: Wallet) => Promise, withdraw: ( dataUnionAddress: string, @@ -34,7 +36,7 @@ const testWithdraw = async ( ) => Promise, requiresMainnetETH: boolean, options: DataUnionWithdrawOptions, -) => { +) { log(`Connecting to Ethereum networks, config = ${JSON.stringify(config)}`) const network = await providerMainnet.getNetwork() log('Connected to "mainnet" network: ', JSON.stringify(network)) @@ -45,7 +47,7 @@ const testWithdraw = async ( const tx1 = await tokenMainnet.mint(adminWalletMainnet.address, parseEther('100')) await tx1.wait() - const adminClient = new StreamrClient(config.clientOptions as any) + const adminClient = new StreamrClient(config.clientOptions) const dataUnion = await adminClient.deployDataUnion() const secret = await dataUnion.createSecret('test secret') @@ -168,7 +170,6 @@ describe('DataUnion withdraw', () => { providerSidechain.removeAllListeners() }) - // TODO: add tests for just getting the hash and doing the transportMessage manually describe.each([ [false, true, true], // sidechain withdraw [true, true, true], // self-service mainnet withdraw @@ -176,12 +177,38 @@ describe('DataUnion withdraw', () => { [true, false, true], // bridge-sponsored mainnet withdraw [true, false, false], // other-sponsored mainnet withdraw ])('Withdrawing with sendToMainnet=%p, payForTransport=%p, wait=%p', (sendToMainnet, payForTransport, waitUntilTransportIsComplete) => { + + // for test debugging: select only one case by uncommenting below, and comment out the above .each block + // const [sendToMainnet, payForTransport, waitUntilTransportIsComplete] = [true, false, true] // bridge-sponsored mainnet withdraw + const options = { sendToMainnet, payForTransport, waitUntilTransportIsComplete } - const getTokenBalance = async (wallet: Wallet) => { + async function getTokenBalance(wallet: Wallet) { return sendToMainnet ? balanceClient.getTokenBalance(wallet.address) : balanceClient.getSidechainTokenBalance(wallet.address) } + // emulate the bridge-sponsored withdrawals + beforeAll(() => { + if (!payForTransport && waitUntilTransportIsComplete) { + const sidechainAmbAddress = '0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64' + const signatureRequestEventSignature = '0x520d2afde79cbd5db58755ac9480f81bc658e5c517fcae7365a3d832590b0183' + providerSidechain.on({ + address: sidechainAmbAddress, + topics: [signatureRequestEventSignature] + }, async (e) => { + const message = defaultAbiCoder.decode(['bytes'], e.data)[0] + const hash = keccak256(message) + const adminClient = new StreamrClient(config.clientOptions) + await adminClient.getDataUnion('0x0000000000000000000000000000000000000000').transportMessage(hash) + }) + } + }) + afterAll(() => { + if (!payForTransport && waitUntilTransportIsComplete) { + providerSidechain.removeAllListeners() + } + }) + describe('by member', () => { it('to itself', () => { From 1e6b1a616dbff70486f86f5c5aff95df8ae5ecb1 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Thu, 22 Apr 2021 13:16:40 +0300 Subject: [PATCH 14/31] cleanup --- test/integration/dataunion/withdraw.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index d7eaee397..1eaf7dfaa 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -190,13 +190,14 @@ describe('DataUnion withdraw', () => { // emulate the bridge-sponsored withdrawals beforeAll(() => { if (!payForTransport && waitUntilTransportIsComplete) { - const sidechainAmbAddress = '0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64' + // event UserRequestForSignature(bytes32 indexed messageId, bytes encodedData) const signatureRequestEventSignature = '0x520d2afde79cbd5db58755ac9480f81bc658e5c517fcae7365a3d832590b0183' + const sidechainAmbAddress = '0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64' providerSidechain.on({ address: sidechainAmbAddress, topics: [signatureRequestEventSignature] - }, async (e) => { - const message = defaultAbiCoder.decode(['bytes'], e.data)[0] + }, async (event) => { + const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // messageId is indexed so it's in topics, only encodedData is in data const hash = keccak256(message) const adminClient = new StreamrClient(config.clientOptions) await adminClient.getDataUnion('0x0000000000000000000000000000000000000000').transportMessage(hash) From cdfedc2b1e43fe74cbb54e9b62ffde0be9ac8b62 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Thu, 22 Apr 2021 13:17:00 +0300 Subject: [PATCH 15/31] longer timeout all tests timed out for some reason; could be that now that there are so many cases (5 x 4, I think) it requires the whole thing to run under 5min, which it might not TODO: turn it down to what's actually needed --- test/integration/dataunion/withdraw.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 1eaf7dfaa..3ca3f4f20 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -162,6 +162,7 @@ async function testWithdraw( } describe('DataUnion withdraw', () => { + jest.setTimeout(3600000) // TODO: remove when it's been figured out how long is really needed const balanceClient = createClient() From 13b20b34afd041f90f83d85051896c98640320ba Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Fri, 23 Apr 2021 09:28:29 +0300 Subject: [PATCH 16/31] timeout tweaks --- test/integration/dataunion/withdraw.test.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 3ca3f4f20..1882d8641 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -162,8 +162,6 @@ async function testWithdraw( } describe('DataUnion withdraw', () => { - jest.setTimeout(3600000) // TODO: remove when it's been figured out how long is really needed - const balanceClient = createClient() afterAll(() => { @@ -219,7 +217,7 @@ describe('DataUnion withdraw', () => { memberClient.getDataUnion(dataUnionAddress).withdrawAll(options) ) return testWithdraw(getBalance, withdraw, true, options) - }, 300000) + }, 3600000) it('to any address', () => { const outsiderWallet = new Wallet(`0x100000000000000000000000000000000000000012300000002${Date.now()}`, providerSidechain) @@ -228,7 +226,7 @@ describe('DataUnion withdraw', () => { memberClient.getDataUnion(dataUnionAddress).withdrawAllTo(outsiderWallet.address, options) ) return testWithdraw(getBalance, withdraw, true, options) - }, 300000) + }, 3600000) }) @@ -240,7 +238,7 @@ describe('DataUnion withdraw', () => { adminClient.getDataUnion(dataUnionAddress).withdrawAllToMember(memberWallet.address, options) ) return testWithdraw(getBalance, withdraw, false, options) - }, 300000) + }, 3600000) it("to anyone with member's signature", async () => { const member2Wallet = new Wallet(`0x100000000000000000000000000040000000000012300000007${Date.now()}`, providerSidechain) @@ -257,7 +255,7 @@ describe('DataUnion withdraw', () => { .withdrawAllToSigned(memberWallet.address, member2Wallet.address, signature, options) } return testWithdraw(getBalance, withdraw, false, options) - }, 300000) + }, 3600000) }) }) From 670377c72b4e9dd8ff998994ab221f60e9ffa3b6 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Fri, 23 Apr 2021 21:27:13 +0300 Subject: [PATCH 17/31] Improve logging bridge timeout probably too low for real world? --- src/dataunion/Contracts.ts | 4 ++-- src/dataunion/DataUnion.ts | 2 +- test/integration/dataunion/withdraw.test.ts | 6 +++++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/dataunion/Contracts.ts b/src/dataunion/Contracts.ts index e8b08920d..9eed1352e 100644 --- a/src/dataunion/Contracts.ts +++ b/src/dataunion/Contracts.ts @@ -12,7 +12,7 @@ import { BigNumber } from '@ethersproject/bignumber' import StreamrEthereum from '../Ethereum' import { StreamrClient } from '../StreamrClient' -const log = debug('StreamrClient::DataUnion') +const log = debug('StreamrClient::Contracts') function validateAddress(name: string, address: EthereumAddress) { if (!isAddress(address)) { @@ -149,7 +149,7 @@ export class Contracts { const [vArray, rArray, sArray]: Todo = [[], [], []] signatures.forEach((signature: string, i) => { - log(` Signature ${i}: ${signature} (len=${signature.length}=${signature.length / 2 - 1} bytes)`) + log(` Signature ${i}: ${signature} (len=${signature.length} = ${signature.length / 2 - 1} bytes)`) rArray.push(signature.substr(2, 64)) sArray.push(signature.substr(66, 64)) vArray.push(signature.substr(130, 2)) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index a348d3a65..d112f95d2 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -624,7 +624,7 @@ export class DataUnion { if (!payForTransport) { // expect someone else to do the transport for us (corresponds to dataUnion.freeWithdraw=true) if (waitUntilTransportIsComplete) { - log(`Waiting for balance ${balanceBefore.toString()} to change`) + log(`Waiting for balance ${balanceBefore.toString()} to change (poll every ${pollingIntervalMs}ms, timeout after ${retryTimeoutMs}ms)`) await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs) .catch((e) => { throw e.message.startsWith('Timeout') ? new Error(`Timeout: Bridge did not transport withdraw message as expected. Fix: DataUnion.transportMessage("${messageHash}")`) : e }) return null diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 1882d8641..2abf0bad7 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -189,6 +189,7 @@ describe('DataUnion withdraw', () => { // emulate the bridge-sponsored withdrawals beforeAll(() => { if (!payForTransport && waitUntilTransportIsComplete) { + log('Starting the simulated bridge-sponsored signature transport process') // event UserRequestForSignature(bytes32 indexed messageId, bytes encodedData) const signatureRequestEventSignature = '0x520d2afde79cbd5db58755ac9480f81bc658e5c517fcae7365a3d832590b0183' const sidechainAmbAddress = '0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64' @@ -196,15 +197,18 @@ describe('DataUnion withdraw', () => { address: sidechainAmbAddress, topics: [signatureRequestEventSignature] }, async (event) => { - const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // messageId is indexed so it's in topics, only encodedData is in data + log(`Observed signature request for message id=${event.topic[1]}`) // messageId is indexed so it's in topics... + const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // ...only encodedData is in data const hash = keccak256(message) const adminClient = new StreamrClient(config.clientOptions) await adminClient.getDataUnion('0x0000000000000000000000000000000000000000').transportMessage(hash) + log(`Transported message hash=${hash}`) }) } }) afterAll(() => { if (!payForTransport && waitUntilTransportIsComplete) { + log('Stopping the simulated bridge-sponsored signature transport process') providerSidechain.removeAllListeners() } }) From acc715778e44fbf1fe2d6e42bea230ccaead6263 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Sat, 24 Apr 2021 20:19:04 +0300 Subject: [PATCH 18/31] these types are in public APIs also TrReceipt -> ContractReceipt was changed in this PR --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 1185e557c..82cd3e739 100644 --- a/src/index.ts +++ b/src/index.ts @@ -28,7 +28,7 @@ export { BigNumber } from '@ethersproject/bignumber' export { ConnectionInfo } from '@ethersproject/web' export { Contract } from '@ethersproject/contracts' export { BytesLike, Bytes } from '@ethersproject/bytes' -export { TransactionReceipt, TransactionResponse } from '@ethersproject/providers' +export { ContractReceipt, ContractTransaction } from '@ethersproject/contracts' export default StreamrClient From 388fd70b477dd22445444784753335a05d3ac767 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Mon, 26 Apr 2021 15:17:09 +0300 Subject: [PATCH 19/31] typo fix --- test/integration/dataunion/withdraw.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 2abf0bad7..47f62250b 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -197,7 +197,7 @@ describe('DataUnion withdraw', () => { address: sidechainAmbAddress, topics: [signatureRequestEventSignature] }, async (event) => { - log(`Observed signature request for message id=${event.topic[1]}`) // messageId is indexed so it's in topics... + log(`Observed signature request for message id=${event.topics[1]}`) // messageId is indexed so it's in topics... const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // ...only encodedData is in data const hash = keccak256(message) const adminClient = new StreamrClient(config.clientOptions) From a502204807f4d29b2e0f4781ad716b270c8b7047 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Mon, 26 Apr 2021 15:42:13 +0300 Subject: [PATCH 20/31] make lint happy making choices based on fairly arbitrary line length limits feels bad man. Adding eslint-disables also feels bad. But I acknowledge it's good to limit line length, in general. I guess. --- src/dataunion/DataUnion.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index d112f95d2..127a1c59b 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -624,9 +624,11 @@ export class DataUnion { if (!payForTransport) { // expect someone else to do the transport for us (corresponds to dataUnion.freeWithdraw=true) if (waitUntilTransportIsComplete) { - log(`Waiting for balance ${balanceBefore.toString()} to change (poll every ${pollingIntervalMs}ms, timeout after ${retryTimeoutMs}ms)`) - await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs) - .catch((e) => { throw e.message.startsWith('Timeout') ? new Error(`Timeout: Bridge did not transport withdraw message as expected. Fix: DataUnion.transportMessage("${messageHash}")`) : e }) + log(`Waiting for balance=${balanceBefore.toString()} change (poll every ${pollingIntervalMs}ms, timeout after ${retryTimeoutMs}ms)`) + await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs).catch((e) => { + const msg = `Timeout: Bridge did not transport withdraw message as expected. Fix: DataUnion.transportMessage("${messageHash}")` + throw e.message.startsWith('Timeout') ? new Error(msg) : e + }) return null } From 8db8d5be429f45a9e50908923060f95437a97bea Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Mon, 26 Apr 2021 16:10:59 +0300 Subject: [PATCH 21/31] clean up old merged PR 241 --- src/dataunion/DataUnion.ts | 5 +++-- test/integration/dataunion/transfer.test.ts | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index fab7ed5ef..ddcfbe5ae 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -501,10 +501,11 @@ export class DataUnion { async transferToMemberInContract( memberAddress: EthereumAddress, amountTokenWei: BigNumber|number|string - ): Promise { + ): Promise { const address = getAddress(memberAddress) // throws if bad address const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) - return duSidechain.transferToMemberInContract(address, amountTokenWei) + const tx = duSidechain.transferToMemberInContract(address, amountTokenWei) + return tx.wait() } /** diff --git a/test/integration/dataunion/transfer.test.ts b/test/integration/dataunion/transfer.test.ts index 726c2b14f..6d8878e13 100644 --- a/test/integration/dataunion/transfer.test.ts +++ b/test/integration/dataunion/transfer.test.ts @@ -122,8 +122,7 @@ describe('DataUnion transfer within contract', () => { await approve.wait() log(`Approve DU ${dataUnion.getSidechainAddress()} to access 1 token from ${adminWalletSidechain.address}`) - const tx = await dataUnion.transferToMemberInContract(memberWallet.address, parseEther('1')) - await tx.wait() + await dataUnion.transferToMemberInContract(memberWallet.address, parseEther('1')) log(`Transfer 1 token with transferWithinContract to ${memberWallet.address}`) const newStats = await memberClient.getDataUnion(dataUnion.getAddress()).getMemberStats(memberWallet.address) From aeded5596dac5ea1d4cbd35588d2769e23366fb0 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Mon, 26 Apr 2021 19:30:03 +0300 Subject: [PATCH 22/31] small fix --- src/dataunion/DataUnion.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index ddcfbe5ae..326d7b814 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -504,7 +504,7 @@ export class DataUnion { ): Promise { const address = getAddress(memberAddress) // throws if bad address const duSidechain = await this.getContracts().getSidechainContract(this.contractAddress) - const tx = duSidechain.transferToMemberInContract(address, amountTokenWei) + const tx = await duSidechain.transferToMemberInContract(address, amountTokenWei) return tx.wait() } From fbed773be030cf58675de5c88df5f7c9cfd62b33 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 27 Apr 2021 10:24:15 +0300 Subject: [PATCH 23/31] deterministic test wallet generation Date.now() feels a bit hacky, running count is better --- test/integration/dataunion/withdraw.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 47f62250b..c4f20d534 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -26,6 +26,8 @@ const tokenMainnet = new Contract(config.clientOptions.tokenAddress, Token.abi, const tokenSidechain = new Contract(config.clientOptions.tokenSidechainAddress, Token.abi, adminWalletSidechain) +let testWalletId = 1000000 // ensure fixed length as string + async function testWithdraw( getBalance: (memberWallet: Wallet) => Promise, withdraw: ( @@ -54,7 +56,8 @@ async function testWithdraw( log(`DataUnion ${dataUnion.getAddress()} is ready to roll`) // dataUnion = await adminClient.getDataUnionContract({dataUnion: "0xd778CfA9BB1d5F36E42526B2BAFD07B74b4066c0"}) - const memberWallet = new Wallet(`0x100000000000000000000000000000000000000012300000001${Date.now()}`, providerSidechain) + testWalletId += 1 + const memberWallet = new Wallet(`0x100000000000000000000000000000000000000012300000000000001${testWalletId}`, providerSidechain) const sendTx = await adminWalletSidechain.sendTransaction({ to: memberWallet.address, value: parseEther('0.1') }) await sendTx.wait() log(`Sent 0.1 sidechain-ETH to ${memberWallet.address}`) From 5a274bf1434973cdb102a0cdc9646cbafb20ff55 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 27 Apr 2021 10:27:56 +0300 Subject: [PATCH 24/31] Use whitelist with bridge-sponsoring All tests run in parallel. Even trying to start bridge "for those cases that need it" ends up bridge paying for other transports as well and every test case being confused. Also for some reason the transport doesn't even happen to the cases that want it. Augh. --- test/integration/dataunion/withdraw.test.ts | 60 +++++++++++---------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index c4f20d534..5e5b643b2 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -137,16 +137,25 @@ async function testWithdraw( const stats = await memberClient.getDataUnion(dataUnion.getAddress()).getMemberStats(memberWallet.address) log(`Stats: ${JSON.stringify(stats)}`) + // "bridge-sponsored mainnet withdraw" case + if (!options.payForTransport && options.waitUntilTransportIsComplete) { + bridgeWhitelist.push(memberWallet.address) + } + // test setup done, do the withdraw const balanceBefore = await getBalance(memberWallet) log(`Balance before: ${balanceBefore}. Withdrawing tokens...`) - let ret = await withdraw(dataUnion.getAddress(), memberClient, memberWallet, adminClient) + + // "other-sponsored mainnet withdraw" case if (typeof ret === 'string') { log(`Transporting message "${ret}"`) ret = await dataUnion.transportMessage(String(ret)) } log(`Tokens withdrawn, return value: ${JSON.stringify(ret)}`) + + // "skip waiting" or "without checking the recipient account" case + // we need to wait nevertheless, to be able to assert that balance in fact changed if (!options.waitUntilTransportIsComplete) { log(`Waiting until balance changes from ${balanceBefore.toString()}`) await until(async () => getBalance(memberWallet).then((b) => !b.eq(balanceBefore))) @@ -164,6 +173,28 @@ async function testWithdraw( expect(balanceIncrease.toString()).toBe(amount.toString()) } +log('Starting the simulated bridge-sponsored signature transport process') +// event UserRequestForSignature(bytes32 indexed messageId, bytes encodedData) +const signatureRequestEventSignature = '0x520d2afde79cbd5db58755ac9480f81bc658e5c517fcae7365a3d832590b0183' +const sidechainAmbAddress = '0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64' +const bridgeWhitelist: string[] = [] +providerSidechain.on({ + address: sidechainAmbAddress, + topics: [signatureRequestEventSignature] +}, async (event) => { + log(`Observed signature request for message id=${event.topics[1]}`) // messageId is indexed so it's in topics... + const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // ...only encodedData is in data + const recipient = '0x' + message.slice(200, 240) + if (bridgeWhitelist.find((address) => address.toLowerCase() === recipient)) { + log(`Recipient ${recipient} not whitelisted, ignoring`) + return + } + const hash = keccak256(message) + const adminClient = new StreamrClient(config.clientOptions) + await adminClient.getDataUnion('0x0000000000000000000000000000000000000000').transportMessage(hash, 100, 120000) + log(`Transported message hash=${hash}`) +}) + describe('DataUnion withdraw', () => { const balanceClient = createClient() @@ -189,33 +220,6 @@ describe('DataUnion withdraw', () => { return sendToMainnet ? balanceClient.getTokenBalance(wallet.address) : balanceClient.getSidechainTokenBalance(wallet.address) } - // emulate the bridge-sponsored withdrawals - beforeAll(() => { - if (!payForTransport && waitUntilTransportIsComplete) { - log('Starting the simulated bridge-sponsored signature transport process') - // event UserRequestForSignature(bytes32 indexed messageId, bytes encodedData) - const signatureRequestEventSignature = '0x520d2afde79cbd5db58755ac9480f81bc658e5c517fcae7365a3d832590b0183' - const sidechainAmbAddress = '0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64' - providerSidechain.on({ - address: sidechainAmbAddress, - topics: [signatureRequestEventSignature] - }, async (event) => { - log(`Observed signature request for message id=${event.topics[1]}`) // messageId is indexed so it's in topics... - const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // ...only encodedData is in data - const hash = keccak256(message) - const adminClient = new StreamrClient(config.clientOptions) - await adminClient.getDataUnion('0x0000000000000000000000000000000000000000').transportMessage(hash) - log(`Transported message hash=${hash}`) - }) - } - }) - afterAll(() => { - if (!payForTransport && waitUntilTransportIsComplete) { - log('Stopping the simulated bridge-sponsored signature transport process') - providerSidechain.removeAllListeners() - } - }) - describe('by member', () => { it('to itself', () => { From 8ebafc3b4fc7a11a69e59d0e63585dc66f714338 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 27 Apr 2021 12:13:57 +0300 Subject: [PATCH 25/31] Fix bridge, refactor out getBalance argument --- test/integration/dataunion/withdraw.test.ts | 61 +++++++++------------ 1 file changed, 25 insertions(+), 36 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 5e5b643b2..a893c2d22 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -13,6 +13,7 @@ import config from '../config' import authFetch from '../../../src/rest/authFetch' import { createClient, createMockAddress, expectInvalidAddress } from '../../utils' import { AmbMessageHash, DataUnionWithdrawOptions, MemberStatus } from '../../../src/dataunion/DataUnion' +import { EthereumAddress } from '../../../src' const log = debug('StreamrClient::DataUnion::integration-test-withdraw') @@ -29,13 +30,13 @@ const tokenSidechain = new Contract(config.clientOptions.tokenSidechainAddress, let testWalletId = 1000000 // ensure fixed length as string async function testWithdraw( - getBalance: (memberWallet: Wallet) => Promise, withdraw: ( - dataUnionAddress: string, + dataUnionAddress: EthereumAddress, memberClient: StreamrClient, memberWallet: Wallet, adminClient: StreamrClient ) => Promise, + recipientAddress: EthereumAddress | null, // null means memberClient itself requiresMainnetETH: boolean, options: DataUnionWithdrawOptions, ) { @@ -73,7 +74,7 @@ async function testWithdraw( auth: { privateKey: memberWallet.privateKey } - } as any) + }) // product is needed for join requests to analyze the DU version const createProductUrl = getEndpointUrl(config.clientOptions.restUrl, 'products') @@ -137,14 +138,21 @@ async function testWithdraw( const stats = await memberClient.getDataUnion(dataUnion.getAddress()).getMemberStats(memberWallet.address) log(`Stats: ${JSON.stringify(stats)}`) + const getRecipientBalance = async () => { + const a = recipientAddress || await memberClient.getAddress() + return options.sendToMainnet ? memberClient.getTokenBalance(a) : memberClient.getSidechainTokenBalance(a) + } + + const balanceBefore = await getRecipientBalance() + log(`Balance before: ${balanceBefore}. Withdrawing tokens...`) + // "bridge-sponsored mainnet withdraw" case if (!options.payForTransport && options.waitUntilTransportIsComplete) { + log(`Adding ${memberWallet.address} to bridge-sponsored withdraw whitelist`) bridgeWhitelist.push(memberWallet.address) } // test setup done, do the withdraw - const balanceBefore = await getBalance(memberWallet) - log(`Balance before: ${balanceBefore}. Withdrawing tokens...`) let ret = await withdraw(dataUnion.getAddress(), memberClient, memberWallet, adminClient) // "other-sponsored mainnet withdraw" case @@ -158,10 +166,10 @@ async function testWithdraw( // we need to wait nevertheless, to be able to assert that balance in fact changed if (!options.waitUntilTransportIsComplete) { log(`Waiting until balance changes from ${balanceBefore.toString()}`) - await until(async () => getBalance(memberWallet).then((b) => !b.eq(balanceBefore))) + await until(async () => getRecipientBalance().then((b) => !b.eq(balanceBefore))) } - const balanceAfter = await getBalance(memberWallet) + const balanceAfter = await getRecipientBalance() const balanceIncrease = balanceAfter.sub(balanceBefore) expect(stats).toMatchObject({ @@ -177,7 +185,7 @@ log('Starting the simulated bridge-sponsored signature transport process') // event UserRequestForSignature(bytes32 indexed messageId, bytes encodedData) const signatureRequestEventSignature = '0x520d2afde79cbd5db58755ac9480f81bc658e5c517fcae7365a3d832590b0183' const sidechainAmbAddress = '0xaFA0dc5Ad21796C9106a36D68f69aAD69994BB64' -const bridgeWhitelist: string[] = [] +const bridgeWhitelist: EthereumAddress[] = [] providerSidechain.on({ address: sidechainAmbAddress, topics: [signatureRequestEventSignature] @@ -185,7 +193,7 @@ providerSidechain.on({ log(`Observed signature request for message id=${event.topics[1]}`) // messageId is indexed so it's in topics... const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // ...only encodedData is in data const recipient = '0x' + message.slice(200, 240) - if (bridgeWhitelist.find((address) => address.toLowerCase() === recipient)) { + if (!bridgeWhitelist.find((address) => address.toLowerCase() === recipient)) { log(`Recipient ${recipient} not whitelisted, ignoring`) return } @@ -196,8 +204,6 @@ providerSidechain.on({ }) describe('DataUnion withdraw', () => { - const balanceClient = createClient() - afterAll(() => { providerMainnet.removeAllListeners() providerSidechain.removeAllListeners() @@ -216,27 +222,19 @@ describe('DataUnion withdraw', () => { const options = { sendToMainnet, payForTransport, waitUntilTransportIsComplete } - async function getTokenBalance(wallet: Wallet) { - return sendToMainnet ? balanceClient.getTokenBalance(wallet.address) : balanceClient.getSidechainTokenBalance(wallet.address) - } - describe('by member', () => { it('to itself', () => { - const getBalance = async (memberWallet: Wallet) => getTokenBalance(memberWallet) - const withdraw = async (dataUnionAddress: string, memberClient: StreamrClient) => ( + return testWithdraw(async (dataUnionAddress, memberClient) => ( memberClient.getDataUnion(dataUnionAddress).withdrawAll(options) - ) - return testWithdraw(getBalance, withdraw, true, options) + ), null, true, options) }, 3600000) it('to any address', () => { const outsiderWallet = new Wallet(`0x100000000000000000000000000000000000000012300000002${Date.now()}`, providerSidechain) - const getBalance = async () => getTokenBalance(outsiderWallet) - const withdraw = (dataUnionAddress: string, memberClient: StreamrClient) => ( + return testWithdraw(async (dataUnionAddress, memberClient) => ( memberClient.getDataUnion(dataUnionAddress).withdrawAllTo(outsiderWallet.address, options) - ) - return testWithdraw(getBalance, withdraw, true, options) + ), outsiderWallet.address, true, options) }, 3600000) }) @@ -244,28 +242,19 @@ describe('DataUnion withdraw', () => { describe('by admin', () => { it('to member without signature', async () => { - const getBalance = async (memberWallet: Wallet) => getTokenBalance(memberWallet) - const withdraw = (dataUnionAddress: string, _: StreamrClient, memberWallet: Wallet, adminClient: StreamrClient) => ( + return testWithdraw(async (dataUnionAddress, memberClient, memberWallet, adminClient) => ( adminClient.getDataUnion(dataUnionAddress).withdrawAllToMember(memberWallet.address, options) - ) - return testWithdraw(getBalance, withdraw, false, options) + ), null, false, options) }, 3600000) it("to anyone with member's signature", async () => { const member2Wallet = new Wallet(`0x100000000000000000000000000040000000000012300000007${Date.now()}`, providerSidechain) - const getBalance = async () => getTokenBalance(member2Wallet) - const withdraw = async ( - dataUnionAddress: string, - memberClient: StreamrClient, - memberWallet: Wallet, - adminClient: StreamrClient - ) => { + return testWithdraw(async (dataUnionAddress, memberClient, memberWallet, adminClient) => { const signature = await memberClient.getDataUnion(dataUnionAddress).signWithdrawAllTo(member2Wallet.address) return adminClient .getDataUnion(dataUnionAddress) .withdrawAllToSigned(memberWallet.address, member2Wallet.address, signature, options) - } - return testWithdraw(getBalance, withdraw, false, options) + }, member2Wallet.address, false, options) }, 3600000) }) }) From f21cd2b88bb64e3a2816da245cc90d110d188efe Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 27 Apr 2021 12:32:18 +0300 Subject: [PATCH 26/31] Fix bridge whitelisting inside the test --- test/integration/dataunion/withdraw.test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index a893c2d22..2373035f0 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -36,7 +36,7 @@ async function testWithdraw( memberWallet: Wallet, adminClient: StreamrClient ) => Promise, - recipientAddress: EthereumAddress | null, // null means memberClient itself + recipientAddress: EthereumAddress | null, // null means memberWallet.address requiresMainnetETH: boolean, options: DataUnionWithdrawOptions, ) { @@ -59,6 +59,7 @@ async function testWithdraw( testWalletId += 1 const memberWallet = new Wallet(`0x100000000000000000000000000000000000000012300000000000001${testWalletId}`, providerSidechain) + const recipient = recipientAddress || memberWallet.address const sendTx = await adminWalletSidechain.sendTransaction({ to: memberWallet.address, value: parseEther('0.1') }) await sendTx.wait() log(`Sent 0.1 sidechain-ETH to ${memberWallet.address}`) @@ -138,18 +139,19 @@ async function testWithdraw( const stats = await memberClient.getDataUnion(dataUnion.getAddress()).getMemberStats(memberWallet.address) log(`Stats: ${JSON.stringify(stats)}`) - const getRecipientBalance = async () => { - const a = recipientAddress || await memberClient.getAddress() - return options.sendToMainnet ? memberClient.getTokenBalance(a) : memberClient.getSidechainTokenBalance(a) - } + const getRecipientBalance = async () => ( + options.sendToMainnet + ? memberClient.getTokenBalance(recipient) + : memberClient.getSidechainTokenBalance(recipient) + ) const balanceBefore = await getRecipientBalance() log(`Balance before: ${balanceBefore}. Withdrawing tokens...`) // "bridge-sponsored mainnet withdraw" case if (!options.payForTransport && options.waitUntilTransportIsComplete) { - log(`Adding ${memberWallet.address} to bridge-sponsored withdraw whitelist`) - bridgeWhitelist.push(memberWallet.address) + log(`Adding ${recipient} to bridge-sponsored withdraw whitelist`) + bridgeWhitelist.push(recipient) } // test setup done, do the withdraw From aebff647449ea94a378efe2c85028cc107a338c9 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Tue, 27 Apr 2021 16:32:31 +0300 Subject: [PATCH 27/31] fix ETH-81: engine-and-editor -> core-api --- .github/workflows/test-code.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-code.yml b/.github/workflows/test-code.yml index 240e1ac88..e7a3added 100644 --- a/.github/workflows/test-code.yml +++ b/.github/workflows/test-code.yml @@ -109,7 +109,7 @@ jobs: - name: Start Streamr Docker Stack uses: streamr-dev/streamr-docker-dev-action@v1.0.0-alpha.3 with: - services-to-start: "mysql redis engine-and-editor cassandra parity-node0 parity-sidechain-node0 bridge broker-node-no-storage-1 broker-node-no-storage-2 broker-node-storage-1 nginx smtp" + services-to-start: "mysql redis core-api cassandra parity-node0 parity-sidechain-node0 bridge broker-node-no-storage-1 broker-node-no-storage-2 broker-node-storage-1 nginx smtp" - name: Run Test run: npm run $TEST_NAME @@ -131,7 +131,7 @@ jobs: - name: Start Streamr Docker Stack uses: streamr-dev/streamr-docker-dev-action@v1.0.0-alpha.3 with: - services-to-start: "mysql redis engine-and-editor cassandra parity-node0 parity-sidechain-node0 bridge broker-node-no-storage-1 broker-node-no-storage-2 broker-node-storage-1 nginx smtp" + services-to-start: "mysql redis core-api cassandra parity-node0 parity-sidechain-node0 bridge broker-node-no-storage-1 broker-node-no-storage-2 broker-node-storage-1 nginx smtp" - uses: nick-invision/retry@v2 name: Run Test with: From b0a6152bfc4686a195e7b60b9452c4db6d063b11 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Wed, 28 Apr 2021 09:05:36 +0300 Subject: [PATCH 28/31] Remove the rest of Date.now() pseudo-ids just to guard against the super improbable failure of shared ids in parallel-run tests --- test/integration/dataunion/withdraw.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index 2373035f0..bc13bfd4b 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -233,7 +233,8 @@ describe('DataUnion withdraw', () => { }, 3600000) it('to any address', () => { - const outsiderWallet = new Wallet(`0x100000000000000000000000000000000000000012300000002${Date.now()}`, providerSidechain) + testWalletId += 1 + const outsiderWallet = new Wallet(`0x100000000000000000000000000000000000000012300000002${testWalletId}`, providerSidechain) return testWithdraw(async (dataUnionAddress, memberClient) => ( memberClient.getDataUnion(dataUnionAddress).withdrawAllTo(outsiderWallet.address, options) ), outsiderWallet.address, true, options) @@ -250,7 +251,8 @@ describe('DataUnion withdraw', () => { }, 3600000) it("to anyone with member's signature", async () => { - const member2Wallet = new Wallet(`0x100000000000000000000000000040000000000012300000007${Date.now()}`, providerSidechain) + testWalletId += 1 + const member2Wallet = new Wallet(`0x100000000000000000000000000040000000000012300000007${testWalletId}`, providerSidechain) return testWithdraw(async (dataUnionAddress, memberClient, memberWallet, adminClient) => { const signature = await memberClient.getDataUnion(dataUnionAddress).signWithdrawAllTo(member2Wallet.address) return adminClient From 9f32fda26e3e6a08d812c5d46fdb140910f7faef Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Fri, 30 Apr 2021 09:45:38 +0300 Subject: [PATCH 29/31] logging overhaul format strings are MUCH nicer for a lot of things: they show the JS type, also pretty-print objects (no need to JSON.stringify) --- test/integration/dataunion/withdraw.test.ts | 75 ++++++++++----------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/test/integration/dataunion/withdraw.test.ts b/test/integration/dataunion/withdraw.test.ts index bc13bfd4b..972f48c86 100644 --- a/test/integration/dataunion/withdraw.test.ts +++ b/test/integration/dataunion/withdraw.test.ts @@ -40,13 +40,13 @@ async function testWithdraw( requiresMainnetETH: boolean, options: DataUnionWithdrawOptions, ) { - log(`Connecting to Ethereum networks, config = ${JSON.stringify(config)}`) + log('Connecting to Ethereum networks, config = %o', config) const network = await providerMainnet.getNetwork() - log('Connected to "mainnet" network: ', JSON.stringify(network)) + log('Connected to "mainnet" network: %o', network) const network2 = await providerSidechain.getNetwork() - log('Connected to sidechain network: ', JSON.stringify(network2)) + log('Connected to sidechain network: %o', network2) - log(`Minting 100 tokens to ${adminWalletMainnet.address}`) + log('Minting 100 tokens to %s', adminWalletMainnet.address) const tx1 = await tokenMainnet.mint(adminWalletMainnet.address, parseEther('100')) await tx1.wait() @@ -54,7 +54,7 @@ async function testWithdraw( const dataUnion = await adminClient.deployDataUnion() const secret = await dataUnion.createSecret('test secret') - log(`DataUnion ${dataUnion.getAddress()} is ready to roll`) + log('DataUnion %s is ready to roll', dataUnion.getAddress()) // dataUnion = await adminClient.getDataUnionContract({dataUnion: "0xd778CfA9BB1d5F36E42526B2BAFD07B74b4066c0"}) testWalletId += 1 @@ -62,12 +62,12 @@ async function testWithdraw( const recipient = recipientAddress || memberWallet.address const sendTx = await adminWalletSidechain.sendTransaction({ to: memberWallet.address, value: parseEther('0.1') }) await sendTx.wait() - log(`Sent 0.1 sidechain-ETH to ${memberWallet.address}`) + log('Sent 0.1 sidechain-ETH to %s', memberWallet.address) if (requiresMainnetETH) { const send2Tx = await adminWalletMainnet.sendTransaction({ to: memberWallet.address, value: parseEther('0.1') }) await send2Tx.wait() - log(`Sent 0.1 mainnet-ETH to ${memberWallet.address}`) + log('Sent 0.1 mainnet-ETH to %s', memberWallet.address) } const memberClient = new StreamrClient({ @@ -89,55 +89,54 @@ async function testWithdraw( }) const res = await memberClient.getDataUnion(dataUnion.getAddress()).join(secret) // await adminClient.addMembers([memberWallet.address], { dataUnion }) - log(`Member joined data union: ${JSON.stringify(res)}`) + log('Member joined data union %o', res) // eslint-disable-next-line no-underscore-dangle const contract = await dataUnion._getContract() const tokenAddress = await contract.token() - log(`Token address: ${tokenAddress}`) + log('Token address: %s', tokenAddress) const adminTokenMainnet = new Contract(tokenAddress, Token.abi, adminWalletMainnet) + async function logBalance(owner: string, address: EthereumAddress) { + const balance = await adminTokenMainnet.balanceOf(address) + log('%s (%s) mainnet token balance: %s (%s)', owner, address, formatEther(balance), balance.toString()) + } const amount = parseEther('1') const duSidechainEarningsBefore = await contract.sidechain.totalEarnings() - const duBalance1 = await adminTokenMainnet.balanceOf(dataUnion.getAddress()) - log(`Token balance of ${dataUnion.getAddress()}: ${formatEther(duBalance1)} (${duBalance1.toString()})`) - const balance1 = await adminTokenMainnet.balanceOf(adminWalletMainnet.address) - log(`Token balance of ${adminWalletMainnet.address}: ${formatEther(balance1)} (${balance1.toString()})`) + await logBalance('Data union', dataUnion.getAddress()) + await logBalance('Admin', adminWalletMainnet.address) - log(`Transferring ${amount} token-wei ${adminWalletMainnet.address}->${dataUnion.getAddress()}`) + log('Transferring %s token-wei %s->%s', amount, adminWalletMainnet.address, dataUnion.getAddress()) const txTokenToDU = await adminTokenMainnet.transfer(dataUnion.getAddress(), amount) await txTokenToDU.wait() - const duBalance2 = await adminTokenMainnet.balanceOf(dataUnion.getAddress()) - log(`Token balance of ${dataUnion.getAddress()}: ${formatEther(duBalance2)} (${duBalance2.toString()})`) - const balance2 = await adminTokenMainnet.balanceOf(adminWalletMainnet.address) - log(`Token balance of ${adminWalletMainnet.address}: ${formatEther(balance2)} (${balance2.toString()})`) + await logBalance('Data union', dataUnion.getAddress()) + await logBalance('Admin', adminWalletMainnet.address) - log(`DU member count: ${await contract.sidechain.activeMemberCount()}`) + log('DU member count: %d', await contract.sidechain.activeMemberCount()) - log(`Transferred ${formatEther(amount)} tokens, next sending to bridge`) + log('Transferred %s tokens, next sending to bridge', formatEther(amount)) const tx2 = await contract.sendTokensToBridge() - await tx2.wait() + const tr2 = await tx2.wait() + log('sendTokensToBridge returned %o', tr2) - log(`Sent to bridge, waiting for the tokens to appear at ${contract.sidechain.address} in sidechain`) + log('Waiting for the tokens to appear at sidechain %s', contract.sidechain.address) await until(async () => !(await tokenSidechain.balanceOf(contract.sidechain.address)).eq('0'), 300000, 3000) - log(`Confirmed tokens arrived, DU balance: ${duSidechainEarningsBefore} -> ${await contract.sidechain.totalEarnings()}`) + log('Confirmed tokens arrived, DU balance: %s -> %s', duSidechainEarningsBefore, await contract.sidechain.totalEarnings()) // make a "full" sidechain contract object that has all functions, not just those required by StreamrClient const sidechainContract = new Contract(contract.sidechain.address, DataUnionSidechain.abi, adminWalletSidechain) const tx3 = await sidechainContract.refreshRevenue() const tr3 = await tx3.wait() - log(`refreshRevenue returned ${JSON.stringify(tr3)}`) - log(`DU balance: ${await contract.sidechain.totalEarnings()}`) + log('refreshRevenue returned %o', tr3) + log('DU sidechain totalEarnings: %o', await contract.sidechain.totalEarnings()) - const duBalance3 = await adminTokenMainnet.balanceOf(dataUnion.getAddress()) - log(`Token balance of ${dataUnion.getAddress()}: ${formatEther(duBalance3)} (${duBalance3.toString()})`) - const balance3 = await adminTokenMainnet.balanceOf(adminWalletMainnet.address) - log(`Token balance of ${adminWalletMainnet.address}: ${formatEther(balance3)} (${balance3.toString()})`) + await logBalance('Data union', dataUnion.getAddress()) + await logBalance('Admin', adminWalletMainnet.address) const stats = await memberClient.getDataUnion(dataUnion.getAddress()).getMemberStats(memberWallet.address) - log(`Stats: ${JSON.stringify(stats)}`) + log('Stats: %o', stats) const getRecipientBalance = async () => ( options.sendToMainnet @@ -146,11 +145,11 @@ async function testWithdraw( ) const balanceBefore = await getRecipientBalance() - log(`Balance before: ${balanceBefore}. Withdrawing tokens...`) + log('Balance before: %s. Withdrawing tokens...', balanceBefore) // "bridge-sponsored mainnet withdraw" case if (!options.payForTransport && options.waitUntilTransportIsComplete) { - log(`Adding ${recipient} to bridge-sponsored withdraw whitelist`) + log('Adding %s to bridge-sponsored withdraw whitelist', recipient) bridgeWhitelist.push(recipient) } @@ -159,15 +158,15 @@ async function testWithdraw( // "other-sponsored mainnet withdraw" case if (typeof ret === 'string') { - log(`Transporting message "${ret}"`) + log('Transporting message "%s"', ret) ret = await dataUnion.transportMessage(String(ret)) } - log(`Tokens withdrawn, return value: ${JSON.stringify(ret)}`) + log('Tokens withdrawn, return value: %o', ret) // "skip waiting" or "without checking the recipient account" case // we need to wait nevertheless, to be able to assert that balance in fact changed if (!options.waitUntilTransportIsComplete) { - log(`Waiting until balance changes from ${balanceBefore.toString()}`) + log('Waiting until balance changes from %s', balanceBefore) await until(async () => getRecipientBalance().then((b) => !b.eq(balanceBefore))) } @@ -192,17 +191,17 @@ providerSidechain.on({ address: sidechainAmbAddress, topics: [signatureRequestEventSignature] }, async (event) => { - log(`Observed signature request for message id=${event.topics[1]}`) // messageId is indexed so it's in topics... + log('Observed signature request for message (id=%s)', event.topics[1]) // messageId is indexed so it's in topics... const message = defaultAbiCoder.decode(['bytes'], event.data)[0] // ...only encodedData is in data const recipient = '0x' + message.slice(200, 240) if (!bridgeWhitelist.find((address) => address.toLowerCase() === recipient)) { - log(`Recipient ${recipient} not whitelisted, ignoring`) + log('Recipient %s not whitelisted, ignoring', recipient) return } const hash = keccak256(message) const adminClient = new StreamrClient(config.clientOptions) await adminClient.getDataUnion('0x0000000000000000000000000000000000000000').transportMessage(hash, 100, 120000) - log(`Transported message hash=${hash}`) + log('Transported message (hash=%s)', hash) }) describe('DataUnion withdraw', () => { From 893409a49bdb91ce9edee2c27864753b347b971e Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Fri, 30 Apr 2021 10:03:57 +0300 Subject: [PATCH 30/31] freeWithdraw -> !payForTransport One is the property of the bridge, one is the property of the function call, it's like the opposite angle of looking at it. But I'll still change it so that the name is the same. So freeWithdraw -> !payForTransport everywhere. It's better that way. --- README.md | 10 +++++----- src/Config.ts | 4 ++-- src/dataunion/DataUnion.ts | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f926d1616..6ea63d4ca 100644 --- a/README.md +++ b/README.md @@ -360,11 +360,11 @@ const dataUnion = client.getDataUnion(dataUnionAddress) ### Admin Functions diff --git a/src/Config.ts b/src/Config.ts index 7ba4bdf5b..6d48b1430 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -65,7 +65,7 @@ export type StrictStreamrClientOptions = { * otherwise the client does the transport as self-service and pays the mainnet gas costs */ minimumWithdrawTokenWei: BigNumber|number|string - freeWithdraw: boolean + payForTransport: boolean factoryMainnetAddress: EthereumAddress factorySidechainAddress: EthereumAddress templateMainnetAddress: EthereumAddress @@ -137,7 +137,7 @@ export const STREAM_CLIENT_DEFAULTS: StrictStreamrClientOptions = { tokenSidechainAddress: '0xE4a2620edE1058D61BEe5F45F6414314fdf10548', dataUnion: { minimumWithdrawTokenWei: '1000000', - freeWithdraw: false, + payForTransport: true, factoryMainnetAddress: '0x7d55f9981d4E10A193314E001b96f72FCc901e40', factorySidechainAddress: '0x1b55587Beea0b5Bc96Bb2ADa56bD692870522e9f', templateMainnetAddress: '0x5FE790E3751dd775Cb92e9086Acd34a2adeB8C7b', diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index 326d7b814..f9a5c72be 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -606,8 +606,8 @@ export class DataUnion { const { pollingIntervalMs = 1000, retryTimeoutMs = 300000, - // by default, transport the signatures if freeWithdraw isn't supported by the sidechain - payForTransport = !this.client.options.dataUnion.freeWithdraw, + // by default, transport the signatures if payForTransport=false isn't supported by the sidechain + payForTransport = this.client.options.dataUnion.payForTransport, waitUntilTransportIsComplete = true, sendToMainnet = true, } = options @@ -637,8 +637,8 @@ export class DataUnion { const messageHash = ambHashes[0] + // expect someone else to do the transport for us if (!payForTransport) { - // expect someone else to do the transport for us (corresponds to dataUnion.freeWithdraw=true) if (waitUntilTransportIsComplete) { log(`Waiting for balance=${balanceBefore.toString()} change (poll every ${pollingIntervalMs}ms, timeout after ${retryTimeoutMs}ms)`) await until(async () => !(await getBalanceFunc()).eq(balanceBefore), retryTimeoutMs, pollingIntervalMs).catch((e) => { @@ -648,7 +648,7 @@ export class DataUnion { return null } - // instead of waiting, hand out the messageHash so that someone else might do the transport using it + // instead of waiting, hand out the messageHash so that we can pass it on to that who does the transportMessage(messageHash) return messageHash } From c0105caa5ba8b6c4cf0992723353309b5b2aed24 Mon Sep 17 00:00:00 2001 From: Juuso Takalainen Date: Fri, 30 Apr 2021 10:11:45 +0300 Subject: [PATCH 31/31] Parallelized some blockchain operations that take an HTTP call --- src/dataunion/DataUnion.ts | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/dataunion/DataUnion.ts b/src/dataunion/DataUnion.ts index f9a5c72be..f256c1679 100644 --- a/src/dataunion/DataUnion.ts +++ b/src/dataunion/DataUnion.ts @@ -135,9 +135,9 @@ export class DataUnion { async isMember(memberAddress: EthereumAddress): Promise { const address = getAddress(memberAddress) const duSidechain = await this.getContracts().getSidechainContractReadOnly(this.contractAddress) - const ACTIVE = 1 // memberData[0] is enum ActiveStatus {None, Active, Inactive} const memberData = await duSidechain.memberData(address) const state = memberData[0] + const ACTIVE = 1 // memberData[0] is enum ActiveStatus {None, Active, Inactive} return (state === ACTIVE) } @@ -298,14 +298,16 @@ export class DataUnion { // TODO: use duSidechain.getMemberStats(address) once it's implemented, to ensure atomic read // (so that memberData is from same block as getEarnings, otherwise withdrawable will be foobar) const duSidechain = await this.getContracts().getSidechainContractReadOnly(this.contractAddress) - const mdata = await duSidechain.memberData(address) - const total = await duSidechain.getEarnings(address).catch(() => BigNumber.from(0)) - const withdrawnEarnings = mdata[3] + const [memberData, total] = await Promise.all([ + duSidechain.memberData(address), + duSidechain.getEarnings(address).catch(() => BigNumber.from(0)), + ]) + const withdrawnEarnings = memberData[3] const withdrawable = total ? total.sub(withdrawnEarnings) : BigNumber.from(0) const STATUSES = [MemberStatus.NONE, MemberStatus.ACTIVE, MemberStatus.INACTIVE] return { - status: STATUSES[mdata[0]], - earningsBeforeLastJoin: mdata[1], + status: STATUSES[memberData[0]], + earningsBeforeLastJoin: memberData[1], totalEarnings: total, withdrawableEarnings: withdrawable, } @@ -669,8 +671,10 @@ export class DataUnion { */ async transportMessage(messageHash: AmbMessageHash, pollingIntervalMs: number = 1000, retryTimeoutMs: number = 300000) { const helper = this.getContracts() - const sidechainAmb = await helper.getSidechainAmb() - const mainnetAmb = await helper.getMainnetAmb() + const [sidechainAmb, mainnetAmb] = await Promise.all([ + helper.getSidechainAmb(), + helper.getMainnetAmb(), + ]) log(`Waiting until sidechain AMB has collected required signatures for hash=${messageHash}...`) await until(async () => helper.requiredSignaturesHaveBeenCollected(messageHash), pollingIntervalMs, retryTimeoutMs) @@ -682,8 +686,10 @@ export class DataUnion { const messageId = '0x' + message.substr(2, 64) log(`Checking mainnet AMB hasn't already processed messageId=${messageId}`) - const alreadySent = await mainnetAmb.messageCallStatus(messageId) - const failAddress = await mainnetAmb.failedMessageSender(messageId) + const [alreadySent, failAddress] = await Promise.all([ + mainnetAmb.messageCallStatus(messageId), + mainnetAmb.failedMessageSender(messageId), + ]) // zero address means no failed messages if (alreadySent || failAddress !== '0x0000000000000000000000000000000000000000') {