Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Limit the number of accounts requested #256

Merged
3 commits merged into from
Sep 3, 2024
Merged

Limit the number of accounts requested #256

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
30e43d4
comment out deprecated and unused "gossip-final-state" handler
thantsintoe Sep 3, 2024
23bd77c
feat: external http have smaller timeouts
kgmyatthu Aug 22, 2024
0ce2a90
Limit the number of accounts requested
jintukumardas Aug 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/network/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ export class NetworkClass extends EventEmitter {
console.log(msg)
this.mainLogger.info('Network: ' + msg)
})
this.extServer.setTimeout(config.network.timeout * 1000)

this.io = require('socket.io')(this.extServer)
resolve(this.io)
Expand Down
34 changes: 21 additions & 13 deletions src/state-manager/AccountSync.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,14 @@
return respond(BadRequest(`${route} invalid request`), serializeResponseError)
}
const readableReq = deserializeGetAccountDataByListReq(requestStream)

// Limit the number of accounts to prevent abuse
const MAX_ACCOUNTS = this.config.stateManager.accountBucketSize // default 200
if (readableReq.accountIds.length > MAX_ACCOUNTS) {
nestedCountersInstance.countEvent('internal', `${route}-too_many_accounts`)
return respond(BadRequest(`${route} too many accounts requested`), serializeResponseError)
}

if (utils.isValidShardusAddress(readableReq.accountIds) === false) {
nestedCountersInstance.countEvent('internal', `${route}-invalid_account_ids`)
return respond(BadRequest(`${route} invalid account_ids`), serializeResponseError)
Expand Down Expand Up @@ -929,7 +937,7 @@
* @param tag a debug tag so that logs and counters will give more context
* @returns GlobalAccountReportResp
*/
async getRobustGlobalReport(tag = '', syncFromArchiver: boolean = false): Promise<GlobalAccountReportResp> {

Check failure on line 940 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

Type boolean trivially inferred from a boolean literal, remove type annotation
console.log('getRobustGlobalReport start', tag, syncFromArchiver)
if (!syncFromArchiver) {
this.lastWinningGlobalReportNodes = []
Expand Down Expand Up @@ -972,20 +980,20 @@
// this.stateManager.config.p2p.useBinarySerializedEndpoints &&
// this.stateManager.config.p2p.getGloablAccountReportBinary
// ) {
const request = {} as GlobalAccountReportReqSerializable
result = await this.p2p.askBinary<
GlobalAccountReportReqSerializable,
GlobalAccountReportRespSerializable
>(
node,
InternalRouteEnum.binary_get_globalaccountreport,
request,
serializeGlobalAccountReportReq,
deserializeGlobalAccountReportResp,
{}
)
const request = {} as GlobalAccountReportReqSerializable
result = await this.p2p.askBinary<

Check failure on line 984 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

'result' is never reassigned. Use 'const' instead
GlobalAccountReportReqSerializable,
GlobalAccountReportRespSerializable
>(
node,
InternalRouteEnum.binary_get_globalaccountreport,
request,
serializeGlobalAccountReportReq,
deserializeGlobalAccountReportResp,
{}
)
// } else {
// result = await this.p2p.ask(node, 'get_globalaccountreport', {})
// result = await this.p2p.ask(node, 'get_globalaccountreport', {})
// }
return checkResultFn(result, node.id)
} catch (error) {
Expand All @@ -1005,7 +1013,7 @@
const signedPayload = this.crypto.sign(payload)
console.log('getGlobalAccountReportFromArchiver messsage', signedPayload)

const getGlobalAccountReportFromArchiver = async () => {

Check failure on line 1016 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

Missing return type on function
const globalAccountReportArchiverUrl = `http://${archiver.ip}:${archiver.port}/get_globalaccountreport_archiver`

try {
Expand All @@ -1019,14 +1027,14 @@
}

let result: Partial<GlobalAccountReportResp> & { msg: string }
result = await getGlobalAccountReportFromArchiver()

Check failure on line 1030 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

'result' is never reassigned. Use 'const' instead
return checkResultFn(result, archiver.publicKey, true)
}

const checkResultFn = (

Check failure on line 1034 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

Missing return type on function
result: (Partial<GlobalAccountReportResp> & { msg: string }) | boolean,
nodeId: string,
resultFromArchiver: boolean = false

Check failure on line 1037 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

Type boolean trivially inferred from a boolean literal, remove type annotation
) => {
// Various failure cases will alter the returned result so that it is tallied in a more orderly way.
// The random numbers were kept to prevent the hash of results from being equal, but now custom equalFn takes care of this concern
Expand Down Expand Up @@ -1369,7 +1377,7 @@
range: StateManagerTypes.shardFunctionTypes.BasicAddressRange,
cycle: number,
initalSync = false,
syncFromArchiver: boolean = false

Check failure on line 1380 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

Type boolean trivially inferred from a boolean literal, remove type annotation
): SyncTrackerInterface {
const index = this.syncTrackerIndex++

Expand All @@ -1393,7 +1401,7 @@
createSyncTrackerByForGlobals(
cycle: number,
initalSync = false,
syncFromArchiver: boolean = false

Check failure on line 1404 in src/state-manager/AccountSync.ts

View workflow job for this annotation

GitHub Actions / ci / QA merge checks 

Type boolean trivially inferred from a boolean literal, remove type annotation
): SyncTrackerInterface {
const index = this.syncTrackerIndex++

Expand Down
165 changes: 83 additions & 82 deletions src/state-manager/TransactionQueue.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,88 +772,89 @@ class TransactionQueue {
}
)

this.p2p.registerGossipHandler(
'gossip-final-state',
async (
payload: { txid: string; stateList: Shardus.WrappedResponse[], txGroupCycle?: number },
sender: Node,
tracker: string,
msgSize: number
) => {
profilerInstance.scopedProfileSectionStart('gossip-final-state', false, msgSize)
const respondSize = cUninitializedSize
try {
// make sure we have it
const queueEntry = this.getQueueEntrySafe(payload.txid) // , payload.timestamp)
//It is okay to ignore this transaction if the txId is not found in the queue.
if (queueEntry == null) {
//In the past we would enqueue the TX, expecially if syncing but that has been removed.
//The normal mechanism of sharing TXs is good enough.
nestedCountersInstance.countEvent('processing', 'gossip-final-state_noQueueEntry')
return
}
if (payload.txGroupCycle) {
if (queueEntry.txGroupCycle !== payload.txGroupCycle) {
/* prettier-ignore */ if (logFlags.error) this.mainLogger.error(`gossip-final-state mismatch txGroupCycle for txid: ${payload.txid}, sender's txGroupCycle: ${payload.txGroupCycle}, our txGroupCycle: ${queueEntry.txGroupCycle}`)
nestedCountersInstance.countEvent(
'processing',
'gossip-final-state: mismatch txGroupCycle for txid ' + payload.txid
)
}
delete payload.txGroupCycle
}
if (logFlags.debug)
this.mainLogger.debug(`gossip-final-state ${queueEntry.logID}, ${Utils.safeStringify(payload.stateList)}`)
// add the data in
let saveSomething = false
for (const data of payload.stateList) {
//let wrappedResponse = data as Shardus.WrappedResponse
//this.queueEntryAddData(queueEntry, data)
if (data == null) {
/* prettier-ignore */ if (logFlags.error && logFlags.verbose) this.mainLogger.error(`broadcast_finalstate data == null`)
continue
}
if (queueEntry.collectedFinalData[data.accountId] == null) {
queueEntry.collectedFinalData[data.accountId] = data
saveSomething = true
/* prettier-ignore */ if (logFlags.playback && logFlags.verbose) this.logger.playbackLogNote('broadcast_finalstate', `${queueEntry.logID}`, `broadcast_finalstate addFinalData qId: ${queueEntry.entryID} data:${utils.makeShortHash(data.accountId)} collected keys: ${utils.stringifyReduce(Object.keys(queueEntry.collectedFinalData))}`)
}

// if (queueEntry.state === 'syncing') {
// /* prettier-ignore */ if (logFlags.playback) this.logger.playbackLogNote('shrd_sync_gotBroadcastfinalstate', `${queueEntry.acceptedTx.txId}`, ` qId: ${queueEntry.entryID} data:${data.accountId}`)
// }
}
if (saveSomething) {
const nodesToSendTo: Set<Node> = new Set()
for (const data of payload.stateList) {
if (data == null) {
continue
}
const storageNodes = this.stateManager.transactionQueue.getStorageGroupForAccount(data.accountId)
for (const node of storageNodes) {
nodesToSendTo.add(node)
}
}
if (nodesToSendTo.size > 0) {
payload.txGroupCycle = queueEntry.txGroupCycle
Comms.sendGossip(
'gossip-final-state',
payload,
undefined,
undefined,
Array.from(nodesToSendTo),
false,
4,
queueEntry.acceptedTx.txId
)
nestedCountersInstance.countEvent(`processing`, `forwarded final data to storage nodes`)
}
}
} finally {
profilerInstance.scopedProfileSectionEnd('gossip-final-state', respondSize)
}
}
)
// THIS HANDLER IS NOT USED ANYMORE
// this.p2p.registerGossipHandler(
// 'gossip-final-state',
// async (
// payload: { txid: string; stateList: Shardus.WrappedResponse[], txGroupCycle?: number },
// sender: Node,
// tracker: string,
// msgSize: number
// ) => {
// profilerInstance.scopedProfileSectionStart('gossip-final-state', false, msgSize)
// const respondSize = cUninitializedSize
// try {
// // make sure we have it
// const queueEntry = this.getQueueEntrySafe(payload.txid) // , payload.timestamp)
// //It is okay to ignore this transaction if the txId is not found in the queue.
// if (queueEntry == null) {
// //In the past we would enqueue the TX, expecially if syncing but that has been removed.
// //The normal mechanism of sharing TXs is good enough.
// nestedCountersInstance.countEvent('processing', 'gossip-final-state_noQueueEntry')
// return
// }
// if (payload.txGroupCycle) {
// if (queueEntry.txGroupCycle !== payload.txGroupCycle) {
// /* prettier-ignore */ if (logFlags.error) this.mainLogger.error(`gossip-final-state mismatch txGroupCycle for txid: ${payload.txid}, sender's txGroupCycle: ${payload.txGroupCycle}, our txGroupCycle: ${queueEntry.txGroupCycle}`)
// nestedCountersInstance.countEvent(
// 'processing',
// 'gossip-final-state: mismatch txGroupCycle for txid ' + payload.txid
// )
// }
// delete payload.txGroupCycle
// }
// if (logFlags.debug)
// this.mainLogger.debug(`gossip-final-state ${queueEntry.logID}, ${Utils.safeStringify(payload.stateList)}`)
// // add the data in
// let saveSomething = false
// for (const data of payload.stateList) {
// //let wrappedResponse = data as Shardus.WrappedResponse
// //this.queueEntryAddData(queueEntry, data)
// if (data == null) {
// /* prettier-ignore */ if (logFlags.error && logFlags.verbose) this.mainLogger.error(`broadcast_finalstate data == null`)
// continue
// }
// if (queueEntry.collectedFinalData[data.accountId] == null) {
// queueEntry.collectedFinalData[data.accountId] = data
// saveSomething = true
// /* prettier-ignore */ if (logFlags.playback && logFlags.verbose) this.logger.playbackLogNote('broadcast_finalstate', `${queueEntry.logID}`, `broadcast_finalstate addFinalData qId: ${queueEntry.entryID} data:${utils.makeShortHash(data.accountId)} collected keys: ${utils.stringifyReduce(Object.keys(queueEntry.collectedFinalData))}`)
// }
//
// // if (queueEntry.state === 'syncing') {
// // /* prettier-ignore */ if (logFlags.playback) this.logger.playbackLogNote('shrd_sync_gotBroadcastfinalstate', `${queueEntry.acceptedTx.txId}`, ` qId: ${queueEntry.entryID} data:${data.accountId}`)
// // }
// }
// if (saveSomething) {
// const nodesToSendTo: Set<Node> = new Set()
// for (const data of payload.stateList) {
// if (data == null) {
// continue
// }
// const storageNodes = this.stateManager.transactionQueue.getStorageGroupForAccount(data.accountId)
// for (const node of storageNodes) {
// nodesToSendTo.add(node)
// }
// }
// if (nodesToSendTo.size > 0) {
// payload.txGroupCycle = queueEntry.txGroupCycle
// Comms.sendGossip(
// 'gossip-final-state',
// payload,
// undefined,
// undefined,
// Array.from(nodesToSendTo),
// false,
// 4,
// queueEntry.acceptedTx.txId
// )
// nestedCountersInstance.countEvent(`processing`, `forwarded final data to storage nodes`)
// }
// }
// } finally {
// profilerInstance.scopedProfileSectionEnd('gossip-final-state', respondSize)
// }
// }
// )

/**
* request_state_for_tx
Expand Down
Loading