From e021983b37eac1468d909219fa49aa998cfc6bfc Mon Sep 17 00:00:00 2001 From: Marcelo Salhab Brogliato Date: Thu, 14 Feb 2019 21:36:02 +0000 Subject: [PATCH] feat(graphviz): Add new tools and refactor previous ones to draw the DAG using graphviz --- Makefile | 2 +- hathor/graphviz.py | 211 ++++++++++++++++++ hathor/loganimation.py | 6 +- hathor/transaction/resources/graphviz.py | 6 +- .../storage/transaction_storage.py | 156 ------------- 5 files changed, 220 insertions(+), 161 deletions(-) create mode 100644 hathor/graphviz.py diff --git a/Makefile b/Makefile index 621740662..2cecfc614 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ tests-cli: .PHONY: tests-lib tests-lib: - pytest --durations=10 $(pytest_flags) --cov-fail-under=94 $(tests_lib) + pytest --durations=10 $(pytest_flags) --cov-fail-under=93 $(tests_lib) .PHONY: tests-simulation tests-simulation: diff --git a/hathor/graphviz.py b/hathor/graphviz.py new file mode 100644 index 000000000..a1c8281bf --- /dev/null +++ b/hathor/graphviz.py @@ -0,0 +1,211 @@ + +from itertools import chain + +from graphviz import Digraph + +from hathor.transaction import BaseTransaction, Block +from hathor.transaction.storage import TransactionStorage + + +def blockchain(tx_storage: TransactionStorage, format: str = 'pdf'): + """ Draw only the blocks and their connections. + It goes through all transactions. So, take care with performance. + """ + dot = Digraph(format=format) + dot.attr(rankdir='RL') + + for tx in tx_storage.get_all_transactions(): + assert tx.hash is not None + + if not tx.is_block: + continue + assert isinstance(tx, Block) + + name = tx.hash.hex() + node_attrs = get_node_attrs(tx) + dot.node(name, **node_attrs) + + if len(tx.parents) > 0: + dot.edge(name, tx.parents[0].hex()) + + return dot + + +def tx_neighborhood(tx: BaseTransaction, format: str = 'pdf', max_level: int = 2) -> Digraph: + """ Draw the blocks and transactions around `tx`. + + :params max_level: Maximum distance between `tx` and the others. + """ + dot = Digraph(format=format) + dot.attr(rankdir='RL') + + dot.attr('node', shape='oval', style='') + # attrs_node = {'label': tx.hash.hex()[-4:]} + + root = tx + to_visit = [(0, tx)] + seen = set([tx.hash]) + + while to_visit: + level, tx = to_visit.pop() + assert tx.hash is not None + assert tx.storage is not None + name = tx.hash.hex() + node_attrs = get_node_attrs(tx) + + if tx.hash == root.hash: + node_attrs.update(dict(style='filled', penwidth='5.0')) + + dot.node(name, **node_attrs) + + meta = tx.get_metadata() + + if level <= max_level: + for h in chain(tx.parents, meta.children): + if h not in seen: + seen.add(h) + tx2 = tx.storage.get_transaction(h) + to_visit.append((level + 1, tx2)) + + for h in tx.parents: + if h in seen: + dot.edge(name, h.hex()) + + return dot + + +def verifications(storage: TransactionStorage, format: str = 'pdf', weight: bool = False, acc_weight: bool = False, + block_only: bool = False) -> Digraph: + """Return a Graphviz object of the DAG of verifications. + + :param format: Format of the visualization (pdf, png, or jpg) + :param weight: Whether to display or not the tx weight + :param acc_weight: Whether to display or not the tx accumulated weight + :return: A Graphviz object + """ + dot = Digraph(format=format) + dot.attr('node', shape='oval', style='') + + g_blocks = dot.subgraph(name='blocks') + g_txs = dot.subgraph(name='txs') + g_genesis = dot.subgraph(name='genesis') + + blocks_set = set() # Set[bytes(hash)] + txs_set = set() # Set[bytes(hash)] + + nodes_iter = storage._topological_sort() + with g_genesis as g_g, g_txs as g_t, g_blocks as g_b: + for i, tx in enumerate(nodes_iter): + assert tx.hash is not None + name = tx.hash.hex() + + attrs_node = get_node_attrs(tx) + attrs_edge = {} + + if block_only and not tx.is_block: + continue + + if tx.is_block: + blocks_set.add(tx.hash) + else: + txs_set.add(tx.hash) + + if weight: + attrs_node.update(dict(label='{}\nw: {:.2f}'.format(attrs_node['label'], tx.weight))) + + if acc_weight: + metadata = tx.get_metadata() + attrs_node.update( + dict(label='{}\naw: {:.2f}'.format(attrs_node['label'], metadata.accumulated_weight))) + + if tx.is_genesis: + g_g.node(name, **attrs_node) + elif tx.is_block: + g_b.node(name, **attrs_node) + else: + g_t.node(name, **attrs_node) + + for parent_hash in tx.parents: + if block_only and parent_hash not in blocks_set: + continue + if parent_hash in blocks_set: + attrs_edge.update(dict(penwidth='3')) + else: + attrs_edge.update(dict(penwidth='1')) + dot.edge(name, parent_hash.hex(), **attrs_edge) + + dot.attr(rankdir='RL') + return dot + + +def funds(storage: TransactionStorage, format: str = 'pdf', weight: bool = False, acc_weight: bool = False) -> Digraph: + """Return a Graphviz object of the DAG of funds. + + :param format: Format of the visualization (pdf, png, or jpg) + :param weight: Whether to display or not the tx weight + :param acc_weight: Whether to display or not the tx accumulated weight + :return: A Graphviz object + """ + dot = Digraph(format=format) + dot.attr('node', shape='oval', style='') + + g_blocks = dot.subgraph(name='blocks') + g_txs = dot.subgraph(name='txs') + g_genesis = dot.subgraph(name='genesis') + + nodes_iter = storage._topological_sort() + with g_genesis as g_g, g_txs as g_t, g_blocks as g_b: + for i, tx in enumerate(nodes_iter): + assert tx.hash is not None + name = tx.hash.hex() + attrs_node = get_node_attrs(tx) + attrs_edge = {} + + if tx.is_block: + attrs_edge.update(dict(penwidth='4')) + + if weight: + attrs_node.update(dict(label='{}\nw: {:.2f}'.format(attrs_node['label'], tx.weight))) + + if acc_weight: + metadata = tx.get_metadata() + attrs_node.update( + dict(label='{}\naw: {:.2f}'.format(attrs_node['label'], metadata.accumulated_weight))) + + if tx.is_genesis: + g_g.node(name, **attrs_node) + elif tx.is_block: + g_b.node(name, **attrs_node) + else: + g_t.node(name, **attrs_node) + + for txin in tx.inputs: + dot.edge(name, txin.tx_id.hex(), **attrs_edge) + + dot.attr(rankdir='RL') + return dot + + +def get_node_attrs(tx: BaseTransaction): + assert tx.hash is not None + + # tx_tips_attrs = dict(style='filled', fillcolor='#F5D76E') + block_attrs = dict(shape='box', style='filled', fillcolor='#EC644B') + + voided_attrs = dict(style='dashed,filled', penwidth='0.25', fillcolor='#BDC3C7') + conflict_attrs = dict(style='dashed,filled', penwidth='2.0', fillcolor='#BDC3C7') + + attrs_node = {'label': tx.hash.hex()[-4:]} + + if tx.is_block: + attrs_node.update(block_attrs) + if tx.is_genesis: + attrs_node.update(dict(fillcolor='#87D37C', style='filled')) + + meta = tx.get_metadata() + if len(meta.voided_by) > 0: + attrs_node.update(voided_attrs) + if tx.hash in meta.voided_by: + attrs_node.update(conflict_attrs) + + return attrs_node diff --git a/hathor/loganimation.py b/hathor/loganimation.py index 6eff5f6ef..c6bfef13b 100644 --- a/hathor/loganimation.py +++ b/hathor/loganimation.py @@ -1,5 +1,6 @@ import os +from hathor import graphviz from hathor.manager import HathorEvents, HathorManager @@ -37,10 +38,11 @@ def on_new_tx(self) -> None: n = self.sequence - dot1 = self.manager.tx_storage.graphviz(format=self.format) + tx_storage = self.manager.tx_storage + dot1 = graphviz.verifications(tx_storage, format=self.format) dot1.render(os.path.join(self.dirname, 'seq_v_{:010d}'.format(n))) - dot2 = self.manager.tx_storage.graphviz_funds(format='png') + dot2 = graphviz.funds(tx_storage, format='png') dot2.render(os.path.join(self.dirname, 'seq_f_{:010d}'.format(n))) self.sequence += 1 diff --git a/hathor/transaction/resources/graphviz.py b/hathor/transaction/resources/graphviz.py index 5ef22cb2b..51260aed5 100644 --- a/hathor/transaction/resources/graphviz.py +++ b/hathor/transaction/resources/graphviz.py @@ -2,6 +2,7 @@ from twisted.web import resource from twisted.web.http import Request +from hathor import graphviz from hathor.api_util import set_cors from hathor.cli.openapi_files.register import register_resource @@ -49,10 +50,11 @@ def _render_GET_thread(self, request: Request) -> bytes: if b'funds' in request.args: funds = self.parseBoolArg(request.args[b'funds'][0].decode('utf-8')) + tx_storage = self.manager.tx_storage if not funds: - dot = self.manager.tx_storage.graphviz(format=dotformat, weight=weight, acc_weight=acc_weight) + dot = graphviz.verifications(tx_storage, format=dotformat, weight=weight, acc_weight=acc_weight) else: - dot = self.manager.tx_storage.graphviz_funds(format=dotformat, weight=weight, acc_weight=acc_weight) + dot = graphviz.funds(tx_storage, format=dotformat, weight=weight, acc_weight=acc_weight) if dotformat == 'dot': request.setHeader(b'content-type', contenttype[dotformat]) diff --git a/hathor/transaction/storage/transaction_storage.py b/hathor/transaction/storage/transaction_storage.py index 01f23cf58..119c35e03 100644 --- a/hathor/transaction/storage/transaction_storage.py +++ b/hathor/transaction/storage/transaction_storage.py @@ -3,7 +3,6 @@ from itertools import chain from typing import Any, Dict, Iterator, List, Optional, Set, Tuple -from graphviz.dot import Digraph from intervaltree.interval import Interval from twisted.internet.defer import inlineCallbacks, succeed @@ -366,161 +365,6 @@ def get_blocks_before(self, hash_bytes: bytes, num_blocks: int = 100) -> List[Bl """ raise NotImplementedError - def graphviz(self, format: str = 'pdf', weight: bool = False, acc_weight: bool = False, - block_only: bool = False) -> Digraph: - """Return a Graphviz object that can be rendered to generate a visualization of the DAG. - - :param format: Format of the visualization (pdf, png, or jpg) - :param weight: Whether to display or not the tx weight - :param acc_weight: Whether to display or not the tx accumulated weight - :return: A Graphviz object - """ - from graphviz import Digraph - - dot = Digraph(format=format) - - g_blocks = dot.subgraph(name='blocks') - g_txs = dot.subgraph(name='txs') - g_genesis = dot.subgraph(name='genesis') - - tx_tips_attrs = dict(style='filled', fillcolor='#F5D76E') - block_attrs = dict(shape='box', style='filled', fillcolor='#EC644B') - - voided_attrs = dict(style='dashed,filled', penwidth='0.25', fillcolor='#BDC3C7') - conflict_attrs = dict(style='dashed,filled', penwidth='2.0', fillcolor='#BDC3C7') - - dot.attr('node', shape='oval', style='') - nodes_iter = self._topological_sort() - - blocks_set = set() # Set[bytes(hash)] - txs_set = set() # Set[bytes(hash)] - - # block_tips = set(x.data for x in self.get_block_tips()) - tx_tips = set(x.data for x in self.get_tx_tips()) - - with g_genesis as g_g, g_txs as g_t, g_blocks as g_b: - for i, tx in enumerate(nodes_iter): - assert tx.hash is not None - name = tx.hash.hex() - attrs_node = {'label': tx.hash.hex()[-4:]} - attrs_edge = {} - - if block_only and not tx.is_block: - continue - - if tx.is_block: - attrs_node.update(block_attrs) - blocks_set.add(tx.hash) - else: - txs_set.add(tx.hash) - - if tx.hash in tx_tips: - attrs_node.update(tx_tips_attrs) - - if weight: - attrs_node.update(dict(label='{}\nw: {:.2f}'.format(attrs_node['label'], tx.weight))) - - if acc_weight: - metadata = tx.get_metadata() - attrs_node.update( - dict(label='{}\naw: {:.2f}'.format(attrs_node['label'], metadata.accumulated_weight))) - - if tx.is_genesis: - attrs_node.update(dict(fillcolor='#87D37C', style='filled')) - g_g.node(name, **attrs_node) - else: - meta = tx.get_metadata() - if len(meta.voided_by) > 0: - attrs_node.update(voided_attrs) - if tx.hash in meta.voided_by: - attrs_node.update(conflict_attrs) - - if tx.is_block: - g_b.node(name, **attrs_node) - else: - g_t.node(name, **attrs_node) - - for parent_hash in tx.parents: - if block_only and parent_hash not in blocks_set: - continue - if parent_hash in blocks_set: - attrs_edge.update(dict(penwidth='3')) - else: - attrs_edge.update(dict(penwidth='1')) - dot.edge(name, parent_hash.hex(), **attrs_edge) - - dot.attr(rankdir='RL') - return dot - - def graphviz_funds(self, format: str = 'pdf', weight: bool = False, acc_weight: bool = False): - """Return a Graphviz object that can be rendered to generate a visualization of the DAG. - - :param format: Format of the visualization (pdf, png, or jpg) - :param weight: Whether to display or not the tx weight - :param acc_weight: Whether to display or not the tx accumulated weight - :return: A Graphviz object - """ - from graphviz import Digraph - - dot = Digraph(format=format) - - g_blocks = dot.subgraph(name='blocks') - g_txs = dot.subgraph(name='txs') - g_genesis = dot.subgraph(name='genesis') - - tx_tips_attrs = dict(style='filled', fillcolor='#F5D76E') - block_attrs = dict(shape='box', style='filled', fillcolor='#EC644B') - - voided_attrs = dict(style='dashed,filled', penwidth='0.25', fillcolor='#BDC3C7') - conflict_attrs = dict(style='dashed,filled', penwidth='2.0', fillcolor='#BDC3C7') - - dot.attr('node', shape='oval', style='') - nodes_iter = self._topological_sort() - - # block_tips = set(x.data for x in self.get_block_tips()) - tx_tips = set(x.data for x in self.get_tx_tips()) - - with g_genesis as g_g, g_txs as g_t, g_blocks as g_b: - for i, tx in enumerate(nodes_iter): - assert tx.hash is not None - name = tx.hash.hex() - attrs_node = {'label': tx.hash.hex()[-4:]} - attrs_edge = {} - - if tx.is_block: - attrs_node.update(block_attrs) - attrs_edge.update(dict(penwidth='4')) - - if tx.hash in tx_tips: - attrs_node.update(tx_tips_attrs) - - if weight: - attrs_node.update(dict(label='{}\nw: {:.2f}'.format(attrs_node['label'], tx.weight))) - - if acc_weight: - metadata = tx.get_metadata() - attrs_node.update( - dict(label='{}\naw: {:.2f}'.format(attrs_node['label'], metadata.accumulated_weight))) - - if tx.is_genesis: - attrs_node.update(dict(fillcolor='#87D37C', style='filled')) - g_g.node(name, **attrs_node) - elif tx.is_block: - g_b.node(name, **attrs_node) - else: - meta = tx.get_metadata() - if len(meta.voided_by) > 0: - attrs_node.update(voided_attrs) - if tx.hash in meta.voided_by: - attrs_node.update(conflict_attrs) - g_t.node(name, **attrs_node) - - for txin in tx.inputs: - dot.edge(name, txin.tx_id.hex(), **attrs_edge) - - dot.attr(rankdir='RL') - return dot - class TransactionStorageAsyncFromSync(TransactionStorage): """Implement async interface from sync interface, for legacy implementations."""