feat: docker containers to replicate evidence hashes

bot-pinner/Dockerfile

@@ -5,6 +5,6 @@ WORKDIR /usr/src/app
 COPY src/requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY src/add_hashes.py .
+COPY src/* .
 
 CMD [ "python", "-u", "./add_hashes.py" ]

bot-pinner/README.md

@@ -1,34 +1,19 @@
-# Kleros Court Data: Containers to replicate and decentralize data
-
-Adapted from @geaxed PoH data container.
+# Kleros Court V2 Evidence: Containers to replicate and decentralize data
 
 ## Quick start
-
-If you don't have a local Ethereum node, add a RPC to the docker-compose file.
-
+If you don't have a local RPC node, add a RPC to the docker-compose file.
 ```
-git clone
-cd
+git clone git@github.com:kleros/kleros-v2.git
+cd bot-pinner/
 docker-compose build
 docker-compose up -d
 ```
 
 ### In the box
-
 1. Standard IPFS container which creates a local mount for data
-2. Custom PoH container that awaits new events and then scrapes the latest hashes and submits it to IPFS.
+2. Evidence container that awaits new events and then scrapes the latest hashes and submits it to IPFS.
 
-## Contributions
 
+## Contributions
 Please visit [contribution.kleros.io](contributing.kleros.io.).
 For any questions, please join the Kleros Discord or Telegram.
-
-## Considerations
-
-⚠️ Alpha version
-
-- Not all data is included yet
-
-⚠️ Full data coverage
-
-- Currently this is serving as a proof of concept, no warranty or support provided at this stage.

bot-pinner/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.4"
 services:
-  ipfs0:
-    container_name: ipfs0
+  ipfs1:
+    container_name: ipfs1
     image: ipfs/go-ipfs:latest
     ports:
 #     - "4001:4001" # ipfs swarm - expose if needed/wanted
@@ -10,13 +10,17 @@ services:
     volumes:
       - ./compose/ipfs0:/data/ipfs
 
-  poh_watcher:
+  kleros-court-v2-evidence:
+      depends_on:
+        - ipfs1
       network_mode:  host
-      container_name: add-poh-hashes
+      container_name: court-v2-evidence
       build: .
       volumes:
-        - ./PoH/:/var/lib/data/
+        - ./court/:/var/lib/data/
+        - ./watchlist/:/var/lib/watchlist/
       environment:
-          RPC: "INFURA/RPC"
-          INTERVAL: 600
+          RPC: "https://rinkeby.arbitrum.io/rpc"
+          INTERVAL: 300
+          RETRY: 2
 

bot-pinner/src/add_hashes.py

@@ -1,77 +1,108 @@
 from web3 import Web3
 import requests
-import json
 import os
 import time
+from tooling import motd, create_logger, port_up
+import json
+import glob
 
-print("Booting...")
 
+logger = create_logger()
+motd()
+
+# Init
 RPC = os.environ.get("RPC", "http://localhost:8545")
-INTERVAL = os.environ.get("INTERVAL", 600)
+INTERVAL = os.environ.get("INTERVAL", 600)  # Events are not constantly listened to, instead it checks per INTERVAL.
+RETRY = int(os.environ.get("RETRY", 0)) # Retry interval value
+attempted_retries = dict()
 
-with open("/var/lib/data/block") as file:
-    block = file.read()
+# Contract / RPC
 w3 = Web3(Web3.HTTPProvider(RPC))
-address = '0xA2c538AA05BBCc44c213441f6f3777223D2BF9e5' # DisputeKitClassic on ArbitrumRinkeby
-address = w3.toChecksumAddress(address)
-abi = '[{"inputs":[{"internalType":"address","name":"_governor","type":"address"},{"internalType":"contract KlerosCore","name":"_core","type":"address"},{"internalType":"contract RNG","name":"_rng","type":"address"}],"stateMutability":"nonpayable","type":"constructor"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"indexed":true,"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"indexed":true,"internalType":"uint256","name":"_choice","type":"uint256"}],"name":"ChoiceFunded","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"indexed":true,"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"_choice","type":"uint256"},{"indexed":true,"internalType":"address","name":"_contributor","type":"address"},{"indexed":false,"internalType":"uint256","name":"_amount","type":"uint256"}],"name":"Contribution","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"contract IArbitrator","name":"_arbitrator","type":"address"},{"indexed":true,"internalType":"uint256","name":"_evidenceGroupID","type":"uint256"},{"indexed":true,"internalType":"address","name":"_party","type":"address"},{"indexed":false,"internalType":"string","name":"_evidence","type":"string"}],"name":"Evidence","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"indexed":true,"internalType":"address","name":"_juror","type":"address"},{"indexed":true,"internalType":"uint256","name":"_choice","type":"uint256"},{"indexed":false,"internalType":"string","name":"_justification","type":"string"}],"name":"Justification","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"internalType":"enum DisputeKitClassic.Phase","name":"_phase","type":"uint8"}],"name":"NewPhaseDisputeKit","type":"event"},{"anonymous":false,"inputs":[{"indexed":true,"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"indexed":true,"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"indexed":false,"internalType":"uint256","name":"_choice","type":"uint256"},{"indexed":true,"internalType":"address","name":"_contributor","type":"address"},{"indexed":false,"internalType":"uint256","name":"_amount","type":"uint256"}],"name":"Withdrawal","type":"event"},{"inputs":[],"name":"LOSER_APPEAL_PERIOD_MULTIPLIER","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"LOSER_STAKE_MULTIPLIER","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"ONE_BASIS_POINT","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"RN","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"RNBlock","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"WINNER_STAKE_MULTIPLIER","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"}],"name":"areCommitsAllCast","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"}],"name":"areVotesAllCast","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256[]","name":"_voteIDs","type":"uint256[]"},{"internalType":"bytes32","name":"_commit","type":"bytes32"}],"name":"castCommit","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256[]","name":"_voteIDs","type":"uint256[]"},{"internalType":"uint256","name":"_choice","type":"uint256"},{"internalType":"uint256","name":"_salt","type":"uint256"},{"internalType":"string","name":"_justification","type":"string"}],"name":"castVote","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_core","type":"address"}],"name":"changeCore","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address payable","name":"_governor","type":"address"}],"name":"changeGovernor","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"contract RNG","name":"_rng","type":"address"}],"name":"changeRandomNumberGenerator","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"core","outputs":[{"internalType":"contract KlerosCore","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"name":"coreDisputeIDToLocal","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256","name":"_numberOfChoices","type":"uint256"},{"internalType":"bytes","name":"_extraData","type":"bytes"},{"internalType":"uint256","name":"_nbVotes","type":"uint256"}],"name":"createDispute","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"}],"name":"currentRuling","outputs":[{"internalType":"uint256","name":"ruling","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"","type":"uint256"}],"name":"disputes","outputs":[{"internalType":"uint256","name":"numberOfChoices","type":"uint256"},{"internalType":"bool","name":"jumped","type":"bool"},{"internalType":"bytes","name":"extraData","type":"bytes"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"disputesWithoutJurors","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"}],"name":"draw","outputs":[{"internalType":"address","name":"drawnAddress","type":"address"}],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"address","name":"_destination","type":"address"},{"internalType":"uint256","name":"_amount","type":"uint256"},{"internalType":"bytes","name":"_data","type":"bytes"}],"name":"executeGovernorProposal","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256","name":"_choice","type":"uint256"}],"name":"fundAppeal","outputs":[],"stateMutability":"payable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256","name":"_coreRoundID","type":"uint256"}],"name":"getCoherentCount","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"internalType":"uint256","name":"_voteID","type":"uint256"}],"name":"getDegreeOfCoherence","outputs":[{"internalType":"uint256","name":"","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"}],"name":"getLastRoundResult","outputs":[{"internalType":"uint256","name":"winningChoice","type":"uint256"},{"internalType":"bool","name":"tied","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"internalType":"uint256","name":"_choice","type":"uint256"}],"name":"getRoundInfo","outputs":[{"internalType":"uint256","name":"winningChoice","type":"uint256"},{"internalType":"bool","name":"tied","type":"bool"},{"internalType":"uint256","name":"totalVoted","type":"uint256"},{"internalType":"uint256","name":"totalCommited","type":"uint256"},{"internalType":"uint256","name":"nbVoters","type":"uint256"},{"internalType":"uint256","name":"choiceCount","type":"uint256"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"internalType":"uint256","name":"_voteID","type":"uint256"}],"name":"getVoteInfo","outputs":[{"internalType":"address","name":"account","type":"address"},{"internalType":"bytes32","name":"commit","type":"bytes32"},{"internalType":"uint256","name":"choice","type":"uint256"},{"internalType":"bool","name":"voted","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"governor","outputs":[{"internalType":"address","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"isResolving","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"internalType":"uint256","name":"_voteID","type":"uint256"}],"name":"isVoteActive","outputs":[{"internalType":"bool","name":"","type":"bool"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"passPhase","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[],"name":"phase","outputs":[{"internalType":"enum DisputeKitClassic.Phase","name":"","type":"uint8"}],"stateMutability":"view","type":"function"},{"inputs":[],"name":"rng","outputs":[{"internalType":"contract RNG","name":"","type":"address"}],"stateMutability":"view","type":"function"},{"inputs":[{"internalType":"uint256","name":"_evidenceGroupID","type":"uint256"},{"internalType":"string","name":"_evidence","type":"string"}],"name":"submitEvidence","outputs":[],"stateMutability":"nonpayable","type":"function"},{"inputs":[{"internalType":"uint256","name":"_coreDisputeID","type":"uint256"},{"internalType":"address payable","name":"_beneficiary","type":"address"},{"internalType":"uint256","name":"_coreRoundID","type":"uint256"},{"internalType":"uint256","name":"_choice","type":"uint256"}],"name":"withdrawFeesAndRewards","outputs":[{"internalType":"uint256","name":"amount","type":"uint256"}],"stateMutability":"nonpayable","type":"function"}]'
-contract = w3.eth.contract(address=address, abi=abi)
+
+# Read block height from disk or pull latest
+block = 0
+try:
+    with open("/var/lib/data/block") as file:
+        block = file.read()
+except FileNotFoundError:
+    block = w3.eth.getBlock("latest")['number']
+
+# Read want-list from disk
+hashes_wanted = list()
+try:
+    with open("/var/lib/data/missed_hashes") as file:
+        hashes_wanted = file.read().splitlines()
+except FileNotFoundError:
+    pass
 
 def main():
     block_number = block
+    tasks = ["Evidence"]
+    contracts = get_contracts()
     while True:
-        tasks = ["Evidence"]
+        logger.info(f"Cycle starting from block #{block_number}. WANTED HASHES: {len(hashes_wanted)}")
         latest = w3.eth.getBlock('latest')['number']
-        for task in tasks:
-            event_filter = eval(str(f'contract.events.{task}.createFilter(fromBlock={block_number}, toBlock={latest})'))
-            for event in event_filter.get_all_entries():
-                try:
-                    if task == "Evidence":
-                            add_evidence(event['transactionHash'].hex())
-                except Exception as e :
-                    print(f"Failure {e} at {block_number}")
-
+        for contract in contracts:
+            for task in tasks:
+                event_filter = eval(str(f'contract.events.{task}.createFilter(fromBlock={block_number}, toBlock={latest})'))
+                for event in event_filter.get_all_entries():
+                    try:
+                        add_hash(event['args']['_evidence'])
+                    except Exception as e:
+                        logger.error(f"Failure {e} at {block_number}")
         block_number = latest
+
+        # Keep track of block height
         with open("/var/lib/data/block", "w") as file:
             file.write(str(latest))
-        time.sleep(int(INTERVAL))
 
+        if len(hashes_wanted) > 0:
+            retry_hashes()
 
-def _get_ipfs_data(uri):
-    url = 'https://ipfs.io/' + uri
-    try:
-        r = requests.get(url)
-    except requests.exceptions.RequestException as e:
-        raise SystemExit(e)
-    return r.content
+        # Persist want-list
+        with open('/var/lib/data/missed_hashes', 'w') as f:
+            for line in hashes_wanted:
+                f.write(f"{line}\n")
+        time.sleep(int(INTERVAL))
 
 
-def add_hash(_hash):
-    r = requests.post(f"http://localhost:5001/api/v0/pin/add/{_hash}")
-    return print(r.content)
-
-
-def add_evidence(tx):
-    def extract_nested(ipfs_uri):
-        ipfs_hashes_list = list()
-        data = _get_ipfs_data(ipfs_uri)
-        j = json.loads(data)
-        file_uri = j['fileURI']
-        nested_hash = json.loads(_get_ipfs_data(file_uri))
-
-        ipfs_hashes_list.append(ipfs_uri)
-        ipfs_hashes_list.append(nested_hash['photo'])
-        ipfs_hashes_list.append(nested_hash['video'])
-        ipfs_hashes_list.append(file_uri)
-        for _hash in ipfs_hashes_list:
-            _hash = _hash.split("/")[2]
+def retry_hashes():
+    for _hash in hashes_wanted:
+        if not _hash in attempted_retries:
+            attempted_retries[_hash] = 0
+        else:
+            attempted_retries[_hash] += 1
+        if RETRY == 0 or attempted_retries[_hash] < RETRY:
             add_hash(_hash)
+        elif attempted_retries[_hash] > int(RETRY + 10): 
+            attempted_retries[_hash] = int(RETRY - 2) # Reset the search
 
-    receipt = w3.eth.getTransaction(tx)
-    decode = contract.decode_function_input(receipt.input)
-    ipfs_uri = dict(decode[1])['_evidence']
-    extract_nested(ipfs_uri)
+def check_hash(_hash):
+    return _hash.rsplit('/', 1)[0] # Recursive pin // i.e. strip _hash/something.json
+
+def add_hash(_hash):
+    _hash = check_hash(_hash)
+    try:
+        r = requests.post(f"http://localhost:5001/api/v0/pin/add/{_hash}", timeout=30)
+        logger.info(f"Added {_hash}")
+        if _hash in hashes_wanted: hashes_wanted.remove(_hash)
+    except requests.exceptions.ReadTimeout:
+        logger.warning(f"Time-out: Couldn't find {_hash} on the IPFS network")
+        if _hash not in hashes_wanted: hashes_wanted.append(_hash)
 
+def get_contracts():
+    contracts = []
+    for f in glob.glob('/var/lib/watchlist/**/*.json', recursive=True):
+        try:
+            with open(f) as fio:
+                data=json.load(fio)
+                abi = data["abi"]
+                address = w3.toChecksumAddress(data["address"])
+                contracts.append(w3.eth.contract(address=address, abi=abi))
+                logger.info(f"Adding to the watchlist: {address}")
+        except FileNotFoundError:
+            pass
+    return contracts
 
 if __name__ == '__main__':
-    main()
+    main()

bot-pinner/src/tooling.py

@@ -0,0 +1,59 @@
+import time
+import socket
+import logging
+import sys
+import glob
+import json
+from web3 import  Web3
+
+def create_logger():
+    # create a logger object
+    logger = logging.getLogger('kleros-v2-evidence-collector')
+    logger.setLevel(logging.INFO)
+    logfile = logging.StreamHandler(sys.stdout)
+    fmt = '%(asctime)s  %(levelname)s  %(message)s'
+    formatter = logging.Formatter(fmt)
+    logfile.setFormatter(formatter)
+    logger.addHandler(logfile)
+    return logger
+
+
+def motd():
+    kleros = """
+            _-_.
+         _-',^. `-_.
+     ._-' ,'   `.   `-_
+    !`-_._________`-':::
+    !   /\        /\::::
+    ;  /  \      /..\:::
+    ! /    \    /....\::
+    !/      \  /......\:
+    ;--.___. \/_.__.--;;
+     '-_    `:!;;;;;;;'
+        `-_, :!;;;''
+            `-!'
+    """
+    print("Booting...")
+    print(kleros)
+    print("Kleros Court V2 Evidence Collector!")
+    time.sleep(10)  # Wait for IPFS to come up
+    print(additional_info())
+
+
+def  additional_info():
+    ipfs_api = port_up(8080)
+    ipfs_gw = port_up(5001)
+    if ipfs_api == 0 and ipfs_gw == 0:
+        return "Gateway and API are up.  IPFS WebUI: http://127.0.0.1:5001/webui"
+    if ipfs_api == 0:
+        return "API is up"
+    return "API or Gateway unavailable. (If running on different ports, disregard this check)"
+
+
+def port_up(port: int, host="127.0.0.1"):
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    result = sock.connect_ex((host, port))
+    sock.close()
+    return result
+
+