Skip to content
Permalink
Browse files

crawl by read EtherScan api

  • Loading branch information...
khaihkd committed Jul 24, 2018
1 parent cc84dd6 commit f2e6138ea7b0cfe1e4c971d95e0d3982c24c2485
Showing with 61 additions and 0 deletions.
  1. +61 −0 crawlEtherScan.js
@@ -0,0 +1,61 @@
'use strict'

const db = require('./models/mongodb')
const request = require('request')
const q = require('./queues')

const mainUrl = 'https://api.etherscan.io/api?module=account&action=tokentx&contractaddress=0x8b353021189375591723e7384262f45709a3c3dc&offset=600&sort=asc&page='

async function processCrawl() {
let page = 1
while (true) {
let url = mainUrl + page.toString()
console.log('Crawl url: ', url)
let b = await crawlUrl(url)
let body = JSON.parse(b)

if (body.status === '0') {
console.log('Finish crawl')
break
}

for (let i=0; i < body.result.length; i++) {
let transaction = body.result[i]

let tran = await new db.Transaction({
hash: transaction.hash,
block: transaction.blockNumber,
fromAccount: transaction.from,
toAccount: transaction.to,
tokenAmount: parseFloat(transaction.value) / 10 ** 18,
isProcess: true
})
tran.save()

await db.Wallet.findOneAndUpdate({address: transaction.from}, {address: transaction.from}, {upsert: true, new: true})
await db.Wallet.findOneAndUpdate({address: transaction.to}, {address: transaction.to}, {upsert: true, new: true})
q.create('subAmountFromWallet', {fromWallet: transaction.from, tokenAmount: parseFloat(transaction.value) / 10 ** 18})
.priority('high').removeOnComplete(true).save()

q.create('addAmountToWallet', {toWallet: transaction.to, tokenAmount: parseFloat(transaction.value) / 10 ** 18})
.priority('high').removeOnComplete(true).save()
}

page++
}
}

function crawlUrl(url) {
return new Promise(function (resolve, reject) {
request(url, function (error, res, body) {
if (!error && res.statusCode == 200) {
resolve(body);
} else {
reject(error);
}
});
});
}


processCrawl()

0 comments on commit f2e6138

Please sign in to comment.
You can’t perform that action at this time.