/
blockr.py
executable file
·139 lines (117 loc) · 5.22 KB
/
blockr.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import boto3
import click
import requests
Session = requests.Session()
def get_account_names(url):
lower_bound_name = 'a'
limit = 1000
accts = list()
try:
while True:
resp = Session.post(url,json={'id':1,'jsonrpc':'2.0','method':'lookup_accounts','params':[lower_bound_name, limit]})
accounts_in_resp = resp.json()['result']
accts.extend(accounts_in_resp)
if lower_bound_name == accounts_in_resp[-1]:
break
lower_bound_name = accounts_in_resp[-1]
except Exception as e:
print(e)
valid_accounts = (a for a in accts if a not in ('',None))
uniq_accounts = set(valid_accounts)
sorted_accounts = sorted(list(uniq_accounts))
return sorted_accounts
def put_json(s3_resource, bucket, key, data):
json_data = json.dumps(data, ensure_ascii=False).encode()
result = s3_resource.Object(bucket, key).put(
Body=json_data, ContentEncoding='UTF-8', ContentType='application/json')
return key, result
def fetch(steemd_url, block_num, method):
if method == 'get_block':
jsonrpc_data = dict(id=block_num,jsonrpc='2.0',method=method,params=[block_num])
if method == 'get_ops_in_block':
jsonrpc_data = dict(id=block_num,jsonrpc='2.0',method=method,params=[block_num,False])
response = Session.post(steemd_url,json=jsonrpc_data)
response.raise_for_status()
response_json = response.json()
response_raw = response.content
assert 'error' not in response_json
assert response_json['id'] == block_num
return response_raw, response_json
@click.group(name='blockr')
@click.option('--bucket', type=click.STRING, default='steemit-dev-sbds')
@click.pass_context
def blockr(ctx, bucket):
"""Steemd blockchain utils"""
ctx.obj = dict(
bucket=bucket,
s3_resource=boto3.resource('s3'),
s3_client=boto3.client('s3'),
region='us-east-1')
@blockr.command(name='list-accounts')
@click.argument('steemd_url', type=click.STRING)
@click.pass_context
def list_accounts(ctx,steemd_url):
accounts = get_account_names(steemd_url)
click.echo(json.dumps(accounts))
@blockr.command(name='put-blocks')
@click.argument('steemd_url', type=click.STRING, default='https://api.steemit.com')
@click.option('--start', type=click.INT, default=4)
@click.option('--end', type=click.INT,default=20000000)
@click.pass_context
def put_blocks(ctx, steemd_url, start, end):
s3_resource = ctx.obj['s3_resource']
bucket = ctx.obj['bucket']
for block_num in range(start, end+1):
try:
raw, block = fetch(steemd_url, block_num, 'get_block')
block_key = '/'.join([str(block_num), 'block.json'])
block = block['result']
_, block_result = put_json(s3_resource, bucket, block_key, block)
click.echo('{block_num}: block result:{block_result}'.format(block_num=block_num,block_result=block_result), err=True)
except Exception as e:
click.echo(str(e),err=True)
@blockr.command(name='put-ops')
@click.argument('steemd_url', type=click.STRING, default='https://api.steemit.com')
@click.option('--start', type=click.INT, default=1)
@click.option('--end', type=click.INT,default=20000000)
@click.pass_context
def put_ops(ctx, steemd_url, start, end):
s3_resource = ctx.obj['s3_resource']
bucket = ctx.obj['bucket']
for block_num in range(start, end+1):
try:
raw, ops = fetch(steemd_url, block_num, 'get_ops_in_block')
ops_key = '/'.join([str(block_num), 'ops.json'])
ops = ops['result']
_, ops_result = put_json(s3_resource, bucket, ops_key, ops)
click.echo('{block_num}: ops result:{ops_result}'.format(block_num=block_num,ops_result=ops_result), err=True)
except Exception as e:
click.echo(str(e),err=True)
@blockr.command(name='put-blocks-and-ops')
@click.argument('steemd_url', type=click.STRING, default='https://api.steemit.com')
@click.option('--start', type=click.INT, default=1)
@click.option('--end', type=click.INT,default=20000000)
@click.pass_context
def put_blocks_and_ops(ctx, steemd_url, start, end):
s3_resource = ctx.obj['s3_resource']
bucket = ctx.obj['bucket']
for block_num in range(start, end+1):
try:
raw, block = fetch(steemd_url, block_num, 'get_block')
block_key = '/'.join([str(block_num), 'block.json'])
block = block['result']
_, block_result = put_json(s3_resource, bucket, block_key, block)
click.echo('{block_num}: block result:{block_result}'.format(block_num=block_num,block_result=block_result), err=True)
raw, ops = fetch(steemd_url, block_num, 'get_ops_in_block')
ops_key = '/'.join([str(block_num), 'ops.json'])
ops = ops['result']
_, ops_result = put_json(s3_resource, bucket, ops_key, ops)
click.echo('{block_num}: ops result:{ops_result}'.format(block_num=block_num,ops_result=ops_result), err=True)
except Exception as e:
click.echo(str(e),err=True)
if __name__ == '__main__':
blockr()
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4