Skip to content
Permalink
Browse files

Merge pull request #6 from khaihkd/master

new crawl & update crawl process
  • Loading branch information...
khaihkd committed Jul 24, 2018
2 parents cc0974e + f2e6138 commit 52bd9f67316d55f576b071346cd1d63c0678cb47
@@ -1 +1,23 @@
# erc20-coin-genesis-swap
# erc20 Tomocoin genesis swap

## Requirement
- MongoDB
- NodeJS
- Redis

## Config
```
cp config/default.json config/local.json
```
Update environment in`local.json` file

## Install
```
npm install
sudo npm install -g pm2
```

# Run
```
pm2 start crawl.js --watch
```
@@ -3,14 +3,15 @@
"host": "localhost",
"port": 6379,
"password": null,
"prefix": "TomoTokenHolder"
"prefix": "TokenHolder"
},
"db": {
"uri": "mongodb://localhost:27017/tomo-holder"
"uri": "mongodb://localhost:27017/tomo"
},
"provider": {
"ws": "wss://mainnet.infura.io/ws",
"http": "https://testnet.tomochain.com"
},
"tomo_address": "0x8b353021189375591723e7384262f45709a3c3dc"
"tomoAddress": "0x8b353021189375591723e7384262f45709a3c3dc",
"processNumber": 6
}
@@ -6,32 +6,38 @@ const web3 = require('./models/blockchain/chain')

let sleep = (time) => new Promise((resolve) => setTimeout(resolve, time))

async function process() {
async function crawlProcess() {
for (let i = 5168958; i < 5995800 ; i++) {
if (i !== 5168958 && i !== 5169011 && i !== 5169173 && i < 5175169) {
continue
}
if (i % 10 === 0) {
console.log('Sleep 10 seconds')
await sleep(10000)
console.log('Sleep 120 seconds')
await sleep(120000)
}

let block = await web3.eth.getBlock(i);
await new db.Block({
let b = await db.Block.findOne({blockNumber: i})
if (b && b.isProcess) {
continue
}
await db.Block.findOneAndUpdate({blockNumber: block.number}, {
hash: block.hash,
blockNumber: block.number,
transactionCount: block.transactions.length,
parentHash: block.parentHash,
timestamp: block.timestamp
}).save()
timestamp: block.timestamp,
}, { upsert: true, new: true })

console.log("Process block number: " + i);
let listTransactions = await block.transactions
if (listTransactions != null && block != null) {
await q.create('newTransaction', {transactions: listTransactions.toString(), blockNumber: block.number})
.priority('normal').removeOnComplete(true).save()
.attempts(5).backoff({delay: 10000})
.priority('low').removeOnComplete(true).save()
await db.Block.findOneAndUpdate({blockNumber: i}, {isProcess: true}, { upsert: true, new: true })
}
}
}

process()
crawlProcess()
@@ -0,0 +1,61 @@
'use strict'

const db = require('./models/mongodb')
const request = require('request')
const q = require('./queues')

const mainUrl = 'https://api.etherscan.io/api?module=account&action=tokentx&contractaddress=0x8b353021189375591723e7384262f45709a3c3dc&offset=600&sort=asc&page='

async function processCrawl() {
let page = 1
while (true) {
let url = mainUrl + page.toString()
console.log('Crawl url: ', url)
let b = await crawlUrl(url)
let body = JSON.parse(b)

if (body.status === '0') {
console.log('Finish crawl')
break
}

for (let i=0; i < body.result.length; i++) {
let transaction = body.result[i]

let tran = await new db.Transaction({
hash: transaction.hash,
block: transaction.blockNumber,
fromAccount: transaction.from,
toAccount: transaction.to,
tokenAmount: parseFloat(transaction.value) / 10 ** 18,
isProcess: true
})
tran.save()

await db.Wallet.findOneAndUpdate({address: transaction.from}, {address: transaction.from}, {upsert: true, new: true})
await db.Wallet.findOneAndUpdate({address: transaction.to}, {address: transaction.to}, {upsert: true, new: true})
q.create('subAmountFromWallet', {fromWallet: transaction.from, tokenAmount: parseFloat(transaction.value) / 10 ** 18})
.priority('high').removeOnComplete(true).save()

q.create('addAmountToWallet', {toWallet: transaction.to, tokenAmount: parseFloat(transaction.value) / 10 ** 18})
.priority('high').removeOnComplete(true).save()
}

page++
}
}

function crawlUrl(url) {
return new Promise(function (resolve, reject) {
request(url, function (error, res, body) {
if (!error && res.statusCode == 200) {
resolve(body);
} else {
reject(error);
}
});
});
}


processCrawl()
@@ -14,6 +14,14 @@ const config = require('config')
*/

const provider = new Web3.providers.WebsocketProvider(config.get('provider.ws'))
provider.connection.onerror(function (e) {
console.log('Provider error: ', e)
process.exit(1)
})
provider.connection.onclose(function (e) {
console.log('End provider: ', e)
proccess.exit(1)
})
const chain = new Web3(provider)

module.exports = chain
@@ -398,7 +398,7 @@ const ABI = [
Tomochain testnet address: 0x76562af804a46ad4c441aca5693583b69b1c387d
*/

const ADDRESS = config.get('tomo_address')
const ADDRESS = config.get('tomoAddress')

const TomoCoin = new chain.eth.Contract(ABI, ADDRESS)

@@ -23,6 +23,9 @@ const Block = new Schema({
},
timestamp: {
type: Number
},
isProcess: {
type: Boolean
}
}, { timestamps: false })

@@ -22,6 +22,9 @@ const Transaction = new Schema({
},
tokenAmount: {
type: Number
},
isProcess: {
type: Boolean
}
}, { timestamps: false })

@@ -2,18 +2,18 @@
"name": "tomo-token-holder",
"version": "1.0.0",
"description": "",
"main": "index.js",
"main": "crawl.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"devDependencies": {
},
"license": "MIT",
"devDependencies": {},
"dependencies": {
"kue": "^0.11.6",
"config": "^1.30.0",
"kue": "^0.11.6",
"mongoose": "^5.2.4",
"request": "^2.87.0",
"web3": "^1.0.0-beta.34"
}
}
@@ -24,15 +24,20 @@ consumer.task = async function(job, done) {

for (let i = 0; i < logs.length; i++) {
let log = logs[i]
if (log.address.toLowerCase() === config.get('tomo_address')){
if (log.address.toLowerCase() === config.get('tomoAddress')){
// TODO: hashcode because this is for ERC20 transfer function
if (log.topics[0].toLowerCase() === '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef') {
let fromWallet = log.topics[1].replace('0x000000000000000000000000', '0x')
let toWallet = log.topics[2].replace('0x000000000000000000000000', '0x')
let tokenAmount = web3.utils.hexToNumberString(log.data) / 10 ** 18
console.log(' - Found Transfer', tokenAmount, ' TOMO from: ', fromWallet, ' to: ', toWallet)

// Make new transaction
let tran = await db.Transaction.findOne({hash: transaction.transactionHash, fromAccount: fromWallet})
if (tran && tran.isProcess) {
continue
}

// Update or make new transaction
await db.Transaction.findOneAndUpdate(
{hash: transaction.transactionHash, fromAccount: fromWallet, toAccount: toWallet},
{
@@ -49,10 +54,17 @@ consumer.task = async function(job, done) {

const q = require('./index')
await q.create('addAmountToWallet', {toWallet: toWallet, tokenAmount: tokenAmount})
.priority('high').removeOnComplete(true).save()
.attempts(5).backoff({delay: 10000})
.priority('critical').removeOnComplete(true).save()

await q.create('subAmountFromWallet', {fromWallet: fromWallet, tokenAmount: tokenAmount})
.priority('high').removeOnComplete(true).save()
.attempts(5).backoff({delay: 10000})
.priority('critical').removeOnComplete(true).save()

await db.Transaction.findOneAndUpdate(
{hash: transaction.transactionHash, fromAccount: fromWallet, toAccount: toWallet},
{isProcess: true}, { upsert: true, new: true }
)

}
}
@@ -0,0 +1,10 @@
'use strict'

const db = require('./models/mongodb')
const q = require('./queues')
const web3 = require('./models/blockchain/chain')


web3.eth.subscribe('logs', {address: config.get('tomoAddress')}, {}, function (error, sync) {
console.log(error, sync)
})

0 comments on commit 52bd9f6

Please sign in to comment.
You can’t perform that action at this time.