Feat/sortition modularity

.github/workflows/contracts-testing.yml

@@ -21,7 +21,7 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Cache node modules
-      uses: actions/cache@v3.0.5
+      uses: actions/cache@v3.0.7
       env:
         cache-name: cache-node-modules
       with:

.github/workflows/deploy-bots.yml

@@ -0,0 +1,24 @@
+on: [workflow_dispatch]
+
+jobs:
+  build-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: aws-actions/setup-sam@v1
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.STAGING_AWS_ACCESS_KEY }}
+          aws-secret-access-key: ${{ secrets.STAGING_AWS_SECRET_KEY }}
+          aws-region: us-east-2
+
+      # sam build 
+      - run: sam build --use-container
+        working-directory: bots
+
+      # Run Unit tests- Specify unit tests here 
+
+      # sam deploy
+      - run: sam deploy --no-confirm-changeset --no-fail-on-empty-changeset --stack-name kleros-v2-bots --s3-bucket kleros-v2-bots --capabilities CAPABILITY_IAM --region us-east-2
+        working-directory: bots

README.md

@@ -10,6 +10,7 @@
 
 <p align="center">
   <a href="https://app.netlify.com/sites/kleros-v2/deploys"><img src="https://api.netlify.com/api/v1/badges/86d94ae8-f655-46a4-a859-d68696173f3a/deploy-status" alt="Netlify Build Status"></a>
+  <a href="https://www.gitpoap.io/gh/kleros/kleros-v2"><img src="https://public-api.gitpoap.io/v1/repo/kleros/kleros-v2/badge" alt="GitPoap badge"></a>
   </br>
   <a href="https://sonarcloud.io/summary/new_code?id=kleros_kleros-v2"><img src="https://sonarcloud.io/api/project_badges/measure?project=kleros_kleros-v2&metric=security_rating" alt="Security Rating"></a>
   <a href="https://sonarcloud.io/summary/new_code?id=kleros_kleros-v2"><img src="https://sonarcloud.io/api/project_badges/measure?project=kleros_kleros-v2&metric=alert_status" alt="Quality Gate Status"></a>

bot-pinner/.gitignore

@@ -0,0 +1,24 @@
+court
+compose
+
+# DAppNodeSDK release directories
+build_*
+releases.json
+
+### yarn ###
+# https://yarnpkg.com/advanced/qa#which-files-should-be-gitignored
+
+# .yarn/unplugged and .yarn/build-state.yml should likely always be ignored since
+# they typically hold machine-specific build artifacts. Ignoring them might however
+# prevent Zero-Installs from working (to prevent this, set enableScripts to false).
+.yarn/unplugged
+.yarn/build-state.yml
+
+# .yarn/cache and .pnp.* may be safely ignored, but you'll need to run yarn install
+# to regenerate them between each branch switch.
+# Uncomment the following lines if you're not using Zero-Installs:
+# .yarn/cache
+# .pnp.*
+
+# End of https://www.toptal.com/developers/gitignore/api/yarn
+

bot-pinner/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY src/* .
+
+CMD [ "python", "-u", "./add_hashes.py" ]

bot-pinner/Dockerfile-dappnode

@@ -0,0 +1,10 @@
+FROM python:3
+
+WORKDIR /usr/src/app
+
+COPY requirements.txt src/* ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY watchlist/* /var/lib/watchlist/
+
+CMD [ "python", "-u", "./add_hashes.py" ]

bot-pinner/README.md

@@ -0,0 +1,85 @@
+# Kleros Court V2 Evidence: Containers to replicate and decentralize data
+
+## Quick start
+
+If you don't have a local RPC node, add a RPC to the docker-compose file.
+
+```bash
+$ git clone git@github.com:kleros/kleros-v2.git
+$ cd bot-pinner/
+$ docker-compose build
+$ docker-compose up -d
+```
+
+### In the box
+
+1. Standard IPFS container which creates a local mount for data
+2. Evidence container that awaits new events and then scrapes the latest hashes and submits it to IPFS.
+
+## DappNode
+
+:warning: For the following steps, you need access to [a DappNode](https://dappnode.io) system with the IPFS service running.
+
+The IPFS service is expected by default at `http://ipfs-cluster.dappnode:9094`, it can be configured at build time in `docker-compose-dappnode.yml` or at runtime in the Config tab of the package in the dAppnode UI.
+
+```bash
+# first time only
+$ yarn install
+```
+
+### Package installation without publishing
+
+```bash
+$ yarn build:dappnode
+  ✔ Verify connection
+  ✔ Create release dir
+  ✔ Validate files
+  ✔ Copy files
+  ✔ Build docker image
+  ✔ Save and compress image
+  ✔ Upload release to IPFS node
+  ✔ Save upload results
+
+  DNP (DAppNode Package) built and uploaded
+  Release hash : /ipfs/Qma773qBk3PqTDQk6biRagcjiNutmit2rpKStq5WbH16XJ
+  http://my.dappnode/#/installer/%2Fipfs%2FQma773qBk3PqTDQk6biRagcjiNutmit2rpKStq5WbH16XJ
+```
+
+Then follow the url to install the package, or open the DappStore page and paste the ipfs hash.
+
+![DappNode package](docs/image.png)
+
+### Publishing
+
+```bash
+# Increase the version number, specify either: major, minor, patch.
+$ yarn bump:dappnode patch
+0.1.1
+
+# Upload to IPFS
+$ yarn publish:dappnode patch
+  ✔ Fetch current version from APM (next version: 0.1.1)
+  ✔ Build and upload
+  ✔ Generate transaction
+
+  DNP (DAppNode Package) published (version 0.1.1)
+  Release hash : /ipfs/xxxx
+  http://my.dappnode/#/installer/%2Fipfs%2Fxxxx
+
+  You must execute this transaction in mainnet to publish a new version of this DNP.
+
+    To    : 0x988b3A538b618C7A603e1c11Ab82Cd16dbE28069
+  Value : 0
+  Data  : 0x32ab6afxxxxx
+  Gas   : 1100000
+
+  You can also execute this transaction with Metamask by following this pre-filled link
+
+  https://dappnode.github.io/sdk-publish/?r=data-pinner.kleros.eth&v=0.1.1&h=%2Fipfs%2Fxxxxx
+
+```
+
+## Contributions
+
+Please visit [contribution.kleros.io](contributing.kleros.io.).
+For any questions, please join the Kleros Discord or Telegram.

bot-pinner/dappnode_package.json

@@ -0,0 +1,27 @@
+{
+  "name": "kleros-data-pinner.public.dappnode.eth",
+  "version": "0.1.0",
+  "shortDescription": "IPFS data pinner for Kleros",
+  "description": "IPFS data pinner for Kleros",
+  "type": "service",
+  "author": "Kleros <contact@kleros.io> (https://github.com/kleros)",
+  "contributors": [
+    "JayBuidl <jb@kleros.io> (https://github.com/jaybuidl)",
+    "George <george@kleros.io> (https://github.com/geaxed)"
+  ],
+  "categories": [
+    "Storage",
+    "Developer tools"
+  ],
+  "links": {
+    "homepage": "https://kleros.io"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kleros/kleros-v2.git"
+  },
+  "bugs": {
+    "url": "https://github.com/kleros/kleros-v2/issues"
+  }
+}

bot-pinner/docker-compose-dappnode.yml

@@ -0,0 +1,17 @@
+version: "3.5"
+services:
+  data-pinner.kleros.eth:
+    build:
+      context: .
+      dockerfile: Dockerfile-dappnode
+    image: "data-pinner.kleros.eth.kleros-data-pinner.public.dappnode.eth:0.1.0"
+    restart: unless-stopped
+    volumes:
+      - "data:/var/lib/data/"
+    environment:
+      RPC: "https://rinkeby.arbitrum.io/rpc"
+      IPFS: "http://ipfs-cluster.dappnode:9094"
+      INTERVAL: 60
+      RETRY: 2
+volumes:
+  data: {}

bot-pinner/docker-compose.yml

@@ -0,0 +1,27 @@
+version: "3.4"
+services:
+  ipfs1:
+    container_name: ipfs1
+    image: ipfs/go-ipfs:latest
+    ports:
+#     - "4001:4001" # ipfs swarm - expose if needed/wanted
+        - "5001:5001" # ipfs api - expose if needed/wanted
+        - "8080:8080" # ipfs gateway - expose if needed/wanted
+    volumes:
+      - ./compose/ipfs0:/data/ipfs
+
+  kleros-court-v2-evidence:
+      depends_on:
+        - ipfs1
+      network_mode:  host
+      container_name: court-v2-evidence
+      build: .
+      volumes:
+        - ./court/:/var/lib/data/
+        - ./watchlist/:/var/lib/watchlist/
+      environment:
+          RPC: "https://rinkeby.arbitrum.io/rpc"
+          IPFS: "http://localhost:5001/api/v0"
+          INTERVAL: 300
+          RETRY: 2
+

bot-pinner/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@kleros/kleros-v2-bot-pinner",
+  "version": "0.1.0",
+  "description": "Pinning of the court data to decentralized storage.",
+  "author": "Kleros",
+  "license": "MIT",
+  "packageManager": "yarn@3.1.0",
+  "volta": {
+    "node": "16.13.0"
+  },
+  "scripts": {
+    "build:local": "docker-compose build",
+    "build:dappnode": "dappnodesdk build --compose_file_name docker-compose-dappnode.yml",
+    "bump:dappnode": "dappnodesdk increase --compose_file_name docker-compose-dappnode.yml",
+    "publish:dappnode": "dappnodesdk publish --compose_file_name docker-compose-dappnode.yml",
+    "run:local": "docker-compose up"
+  },
+  "engines": {
+    "node": ">=16.13.0"
+  },
+  "devDependencies": {
+    "@dappnode/dappnodesdk": "^0.2.65"
+  }
+}

bot-pinner/requirements.txt

@@ -0,0 +1,2 @@
+web3
+requests

bot-pinner/src/add_hashes.py

@@ -0,0 +1,109 @@
+from web3 import Web3
+import requests
+import os
+import time
+from tooling import motd, create_logger, port_up
+import json
+import glob
+
+
+logger = create_logger()
+motd()
+
+# Init
+RPC = os.environ.get("RPC", "http://localhost:8545")
+IPFS = os.environ.get("IPFS", "http://ipfs-cluster.dappnode:9094")
+INTERVAL = os.environ.get("INTERVAL", 600)  # Events are not constantly listened to, instead it checks per INTERVAL.
+RETRY = int(os.environ.get("RETRY", 0)) # Retry interval value
+attempted_retries = dict()
+
+# Contract / RPC
+w3 = Web3(Web3.HTTPProvider(RPC))
+
+# Read block height from disk or pull latest
+block = 0
+try:
+    with open("/var/lib/data/block") as file:
+        block = file.read()
+except FileNotFoundError:
+    block = w3.eth.getBlock("latest")['number']
+
+# Read want-list from disk
+hashes_wanted = list()
+try:
+    with open("/var/lib/data/missed_hashes") as file:
+        hashes_wanted = file.read().splitlines()
+except FileNotFoundError:
+    pass
+
+def main():
+    block_number = block
+    tasks = ["Evidence"]
+    contracts = get_contracts()
+    while True:
+        logger.info(f"Cycle starting from block #{block_number}. WANTED HASHES: {len(hashes_wanted)}")
+        latest = w3.eth.getBlock('latest')['number']
+        for contract in contracts:
+            for task in tasks:
+                event_filter = eval(str(f'contract.events.{task}.createFilter(fromBlock={block_number}, toBlock={latest})'))
+                for event in event_filter.get_all_entries():
+                    try:
+                        add_hash(event['args']['_evidence'])
+                    except Exception as e:
+                        logger.error(f"Failure {e} at {block_number}")
+        block_number = latest
+
+        # Keep track of block height
+        with open("/var/lib/data/block", "w") as file:
+            file.write(str(latest))
+
+        if len(hashes_wanted) > 0:
+            retry_hashes()
+
+        # Persist want-list
+        with open('/var/lib/data/missed_hashes', 'w') as f:
+            for line in hashes_wanted:
+                f.write(f"{line}\n")
+        time.sleep(int(INTERVAL))
+
+
+def retry_hashes():
+    for _hash in hashes_wanted:
+        if not _hash in attempted_retries:
+            attempted_retries[_hash] = 0
+        else:
+            attempted_retries[_hash] += 1
+        if RETRY == 0 or attempted_retries[_hash] < RETRY:
+            add_hash(_hash)
+        elif attempted_retries[_hash] > int(RETRY + 10): 
+            attempted_retries[_hash] = int(RETRY - 2) # Reset the search
+
+def check_hash(_hash):
+    return _hash.rsplit('/', 1)[0] # Recursive pin // i.e. strip _hash/something.json
+
+def add_hash(_hash):
+    _hash = check_hash(_hash)
+    try:
+        r = requests.post(f"{IPFS}/pin/add/{_hash}", timeout=30)
+        logger.info(f"Added {_hash}")
+        if _hash in hashes_wanted: hashes_wanted.remove(_hash)
+    except requests.exceptions.ReadTimeout:
+        logger.warning(f"Time-out: Couldn't find {_hash} on the IPFS network")
+        if _hash not in hashes_wanted: hashes_wanted.append(_hash)
+
+def get_contracts():
+    contracts = []
+    for f in glob.glob('/var/lib/watchlist/**/*.json', recursive=True):
+        try:
+            with open(f) as fio:
+                data=json.load(fio)
+                abi = data["abi"]
+                address = w3.toChecksumAddress(data["address"])
+                contracts.append(w3.eth.contract(address=address, abi=abi))
+                logger.info(f"Adding to the watchlist: {address}")
+        except FileNotFoundError:
+            pass
+    return contracts
+
+if __name__ == '__main__':
+    main()