Merge pull request #116 from covalenthq/develop

Rudder-DTM-RC:v0.2.6

.github/workflows/tag-release.yaml

@@ -3,14 +3,30 @@ name: tag-release
 on:
   push:
     tags:
-      - "v*"
+      - "v*.*.*"
 
 jobs:
   tagged-release:
     name: Tagged Release
     runs-on: "ubuntu-latest"
 
     steps:
+      - name: Login to Artifact Registry/GCR
+        uses: docker/login-action@v2
+        with:
+          registry: us-docker.pkg.dev
+          username: _json_key
+          password: ${{ secrets.GCR_JSON_KEY }}
+
+      - uses: actions/checkout@v2
+
+      - name: Set env
+        run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+
+      - name: Build & Publish the Docker image
+        run: |
+          docker buildx create --name builder --use --platform=linux/amd64,linux/arm64 && docker buildx build --platform=linux/amd64,linux/arm64 . -t us-docker.pkg.dev/covalent-project/network/rudder:stable -t us-docker.pkg.dev/covalent-project/network/rudder:"${{ env.TAG }}" --push
+
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:
           repo_token: "${{ secrets.GITHUB_TOKEN }}"

docker-compose-ci.yml

@@ -94,8 +94,8 @@ services:
         done;
         echo Proof-chain contracts deployed!;
         echo Uploading test files to local .ipfs...;
-        curl -F "filedata=@/app/test-data/encoded/1-15892728-replica-0x84a541916d6d3c974b4da961e2f00a082c03a071132f5db27787124597c094c1" http://ipfs-pinner:3000/upload;
-        curl -F "filedata=@/app/test-data/encoded/1-15892755-replica-0x0e7bfa908e3fc04003e00afde7eb31772a012e51e0e094e4add1734da92297f7" http://ipfs-pinner:3000/upload;
+        curl -F "filedata=@/app/test-data/codec-0.35/encoded/1-17090940-replica-0x7b8e1d463a0fbc6fce05b31c5c30e605aa13efaca14a1f3ba991d33ea979b12b" http://ipfs-pinner:3000/upload;
+        curl -F "filedata=@/app/test-data/codec-0.35/encoded/1-17090960-replica-0xc95d44182ee006e79f1352ef32664210f383baa016988d5ab2fd950b52bf22ff" http://ipfs-pinner:3000/upload;
         echo Test bsp files uploaded!;
         cd /app;
         MIX_ENV=docker mix test --trace --slowest 10;

docker-compose-mbase.yml

@@ -3,7 +3,7 @@ version: '3'
 # set .env such that all services in docker are talking to each other only
 services:
   ipfs-pinner:
-    image: "us-docker.pkg.dev/covalent-project/network/ipfs-pinner:latest"
+    image: "us-docker.pkg.dev/covalent-project/network/ipfs-pinner:stable"
     volumes:
       - ~/.ipfs:/root/.ipfs/
     container_name: ipfs-pinner
@@ -20,7 +20,7 @@ services:
       - "3000:3000"
 
   evm-server:
-    image: "us-docker.pkg.dev/covalent-project/network/evm-server:latest"
+    image: "us-docker.pkg.dev/covalent-project/network/evm-server:stable"
     container_name: evm-server
     restart: on-failure
     expose:
@@ -31,7 +31,7 @@ services:
       - "3002:3002"
 
   rudder:
-    image: "us-docker.pkg.dev/covalent-project/network/rudder:latest"
+    image: "us-docker.pkg.dev/covalent-project/network/rudder:stable"
     container_name: rudder
     links:
       - "ipfs-pinner:ipfs-pinner"

docker-compose-mbeam.yml

@@ -3,7 +3,7 @@ version: '3'
 # set .env such that all services in docker are talking to each other only
 services:
   ipfs-pinner:
-    image: "us-docker.pkg.dev/covalent-project/network/ipfs-pinner:latest"
+    image: "us-docker.pkg.dev/covalent-project/network/ipfs-pinner:stable"
     volumes:
       - ~/.ipfs:/root/.ipfs/
     container_name: ipfs-pinner
@@ -20,7 +20,7 @@ services:
       - "3000:3000"
 
   evm-server:
-    image: "us-docker.pkg.dev/covalent-project/network/evm-server:latest"
+    image: "us-docker.pkg.dev/covalent-project/network/evm-server:stable"
     container_name: evm-server
     restart: on-failure
     expose:
@@ -31,7 +31,7 @@ services:
       - "3002:3002"
 
   rudder:
-    image: "us-docker.pkg.dev/covalent-project/network/rudder:latest"
+    image: "us-docker.pkg.dev/covalent-project/network/rudder:stable"
     container_name: rudder
     links:
       - "ipfs-pinner:ipfs-pinner"

docs/CONTRIBUTING.md

@@ -0,0 +1,131 @@
+# Contributing
+
+* [Reporting Bugs](#bugs)
+* [General Procedure](#general_procedure)
+  * [Development Procedure](#dev_procedure)
+  * [Dependencies](#dependencies)
+  * [Testing](#testing)
+  * [Linting](#linting)
+  * [Branching Model and Release](#braching_model_and_release)
+  * [PR Targeting](#pr_targeting)
+  * [Pull Requests](#pull_requests)
+  * [Process for reviewing PRs](#reviewing_prs)
+  * [Pull Merge Procedure](#pull_merge_procedure)
+
+## <span id="bugs">Reporting Bugs</span>
+
+Please file bugs in the [GitHub Issue
+Tracker](https://github.com/covalenthq/rudder). Include at
+least the following:
+
+* What happened.
+* What did you expect to happen instead of what *did* happen, if it's
+   not crazy obvious.
+* What operating system, operating system version and version of
+   `rudder` you are running.
+* Console log entries, where possible and relevant.
+
+If you're not sure whether something is relevant, erring on the side of too much information will never be a cause for concern.
+
+## <span id="general_procedure">General Procedure</span>
+
+Contributing to this repo can mean many things such as participating in discussion or proposing code changes. To ensure a smooth workflow for all contributors, the following general procedure for contributing has been established:
+
+1. Either [open](https://github.com/covalenthq/rudder/issues/new/choose)
+   or [find](https://github.com/covalenthq/rudder/issues) an issue you have identified and would like to contribute to
+   resolving.
+
+2. Participate in thoughtful discussion on that issue.
+
+3. If you would like to contribute:
+    1. If the issue is a proposal, ensure that the proposal has been accepted by the Covalent team.
+    2. Ensure that nobody else has already begun working on the same issue. If someone already has, please make sure to contact the individual to collaborate.
+    3. If nobody has been assigned the issue and you would like to work on it, make a comment on the issue to inform the
+       community of your intentions to begin work. Ideally, wait for confirmation that no one has started it. However,
+       if you are eager and do not get a prompt response, feel free to dive on in!
+    4. Follow standard Github best practices:
+        1. Fork the repo
+        2. Branch from the HEAD of `develop`(For core developers working within the `rudder` repo, to ensure a clear ownership of branches, branches must be named with the convention `{moniker}/{issue#}-branch-name`).
+        3. Make commits
+        4. Submit a PR to `develop`
+    5. Be sure to submit the PR in `Draft` mode. Submit your PR early, even if it's incomplete as this indicates to the community you're working on something and allows them to provide comments early in the development process.
+    6. When the code is complete it can be marked `Ready for Review`.
+    7. Be sure to include a relevant change log entry in the `Unreleased` section of `CHANGELOG.md` (see file for log
+       format).
+    8. Please make sure to run `mix format mix.exs "lib/**/*.{ex,exs}" "test/**/*.{ex,exs}" "config/*.{ex,exs}"` before every commit - the easiest way to do this is having your editor run it for you upon saving a file. Additionally, please ensure that your code is lint compliant by running `mix deps.get, mix deps.compile` .
+   There are CI tests built into the `rudder` repository and all PR’s will require that these tests pass before they are able to be merged.
+
+**Note**: for very small or blatantly obvious problems (such as typos), it is not required to open an issue to submit a PR, but be aware that for more complex problems/features, if a PR is opened before an adequate design discussion has taken place in a github issue, that PR runs a high likelihood of being rejected.
+
+Looking for a good place to start contributing? How about checking out
+some [good first issues](https://github.com/covalenthq/rudder/issues).
+
+### <span id="dev_procedure">Development Procedure</span>
+
+1. The latest state of development is on `main`.
+2. `main` must never
+   fail `mix format --check-formatted, mix credo`
+3. No `--force` onto `main` (except when reverting a broken commit, which should seldom happen).
+4. Create your feature branch from `main` either on `github.com/covalenthq/rudder`, or your fork (
+   using `git remote add origin`).
+5. Before submitting a pull request, begin `git rebase` on top of `main`.
+6. Code must adhere to the official elixir [formatting](https://hexdocs.pm/mix/main/Mix.Tasks.Format.html) guidelines.
+7. Code must be documented adhering to the official Elixir [commentary](https://github.com/christopheradams/elixir_style_guide) guidelines.
+8. Pull requests need to be based on and opened against the `main` branch.
+9. Commit messages should be prefixed with the package(s) they modify.
+   * E.g. "eth, rpc: make trace configs optional"
+
+### <span id="dependencies">Dependencies</span>
+
+We use [Mix](https://elixir-lang.org/getting-started/mix-otp/introduction-to-mix.html) to manage dependency versions.
+
+The main branch of every `rudder` repository should just build with `mix deps.get, mix deps.compile`, which means they should be kept up-to-date with their dependencies, so we can get away with telling people they can just `mix deps.get` our software. Since some dependencies are not under our control, a third party may break our build, in which case we can fall back on `rm -rf _build deps && mix clean && mix deps.get && mix deps.compile`.
+
+### <span id="testing">Testing</span>
+
+Covalent uses [GitHub Actions](https://github.com/features/actions) for automated [integration testing](https://github.com/covalenthq/rudder/actions).
+
+### <spand id="linting">Linting</span>
+
+The repo uses `credo` to run linters and enforce coding standards. There are two ways to run this:
+
+* run `mix credo --all`
+* run `mix credo explain Credo.Check.Readability.ModuleDoc`
+
+### <span id="braching_model_and_release">Branching Model and Release</span>
+
+User-facing repos should adhere to the [trunk based development branching model](https://trunkbaseddevelopment.com/).
+
+Libraries need not follow the model strictly, but would be wise to.
+`rudder` utilizes [semantic versioning](https://semver.org/).
+
+### <span id="pr_targeting">PR Targeting</span>
+
+Ensure that you base and target your PR on the `main` branch.
+All feature additions should be targeted against `main`. Bug fixes for an outstanding release candidate should be
+targeted against the release candidate branch.
+
+### <span id="pull_requests">Pull Requests</span>
+
+To accommodate the review process, we suggest that PRs are categorically broken up. Ideally each PR addresses only a single issue. Additionally, as much as possible code refactoring and cleanup should be submitted as separate PRs from bug fixes/feature-additions.
+
+### <span id="reviewing_prs">Process for reviewing PRs</span>
+
+All PRs require two Reviews before merge. When reviewing PRs, please use the following review explanations:
+
+1. `LGTM` without an explicit approval means that the changes look good, but you haven't pulled down the code, run tests, locally and thoroughly reviewed it.
+2. `Approval` through the GH UI means that you understand the code, documentation/spec is updated in the right places,
+   you have pulled down and tested the code locally. In addition:
+    * You must think through whether any added code could be partially combined (DRYed) with existing code.
+    * You must think through any potential security issues or incentive-compatibility flaws introduced by the changes.
+    * Naming convention must be consistent with the rest of the codebase.
+    * Code must live in a reasonable location, considering dependency structures (e.g. not importing testing modules in
+      production code, or including example code modules in production code).
+    * If you approve of the PR, you are responsible for fixing any of the issues mentioned here.
+3. If you are only making "surface level" reviews, submit any notes as `Comments` without adding a review.
+
+### <span id="pull_merge_procedure">Pull Merge Procedure</span>
+
+1. Ensure pull branch is rebased on `main`.
+2. Ensure that all CI tests and checks pass.
+3. Merge pull request @author.

docs/development.md

@@ -0,0 +1,45 @@
+# Development
+
+## Test
+
+To start mock node, ProofChain, IPFS-Pinner and EVM server in Terminal 1 run:
+
+```bash
+docker compose --env-file '.env' -f 'docker-compose-mbase.yml' up --remove-orphans
+```
+
+Wait a minute then in Terminal 2 run:
+
+```elixir
+mix test
+```
+
+### <span id="rudder_docker_pull">Pull</span>
+
+Pull only the latest containerized version of rudder using the following -
+
+Make sure you're logged into gcr by running
+
+```bash
+gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io
+```
+
+Pull image
+
+```docker
+docker pull us-docker.pkg.dev/covalent-project/network/rudder
+```
+
+### <span id="rudder_docker_env">Environment</span>
+
+Add the env vars to a .env file as below. Ask your node operator about these if you have questions. Check the `.env_example` for the list of required (and optional) environment variables.
+
+**Note**: When passing the private key into the env vars as above please remove the `0x` prefix so the private key env var has exactly 64 characters.
+
+```bash
+export BLOCK_RESULT_OPERATOR_PRIVATE_KEY=block-result-operator-private-key-without-0x-prefix
+export NODE_ETHEREUM_MAINNET="https://moonbeam-alphanet.web3.covalenthq.com/alphanet/direct-rpc"
+export IPFS_PINNER_URL="http://ipfs-pinner:3000"
+export EVM_SERVER_URL="http://evm-server:3002"
+export WEB3_JWT="****"
+```

docs/metrics.md

@@ -0,0 +1,8 @@
+# Metrics collection and reporting
+
+`rudder` enabled metrics collection via `--metrics` flag. A metrics server can be enabled (by `--metrics.addr` and `--metrics.port`). The metrics are served in two formats:
+
+- `/debug/metrics`: json representation of expvars and go-metrics
+- `/debug/metrics/prometheus`: same metrics as above but in prometheus format
+
+Monitoring can be setup (for example) by plugging the endpoint serving in prometheus-format into influxdb, which is plugged into grafana.

docs/rudder-compose.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=rudder docker compose
+PartOf=docker.service
+After=docker.service
+
+[Service]
+User=blockchain
+Group=blockchain
+Environment=HOME=/home/blockchain/tmp
+Environment="BLOCK_RESULT_OPERATOR_PRIVATE_KEY=ba3193ff8df497b5369f0d5c92fe443efd7936ab910084b8d4e1d510f05da1b2"
+Environment="NODE_ETHEREUM_MAINNET=https://moonbeam.web3.com/alphanet/rpc"
+Environment="IPFS_PINNER_URL=http://ipfs-pinner:3000"
+Environment="EVM_SERVER_URL=http://evm-server:3002"
+Environment="WEB3_JWT=iI91eXzhJUCyIJIINC6bsJciVkOIIpniGRc5.UiNjXLQJJYdYE2NfXdN0ziG5zWFeIW3EpjygiLOol1caizyZIz5WkMTiMMzT1MAZNM3ITmCQGcLNbeJOMQb43JOiwQ3TQMJz2zTMWECOnsIQhWdNoi6ZipYtc2XOMixAW3JQR4cwlTNkYgJDmhjiJQFYkQ3ciUbWOR.2Vx73BD3BWoD5FG6alOp7foK8krI7Akysr5lVbhP4Bu"
+Type=simple
+ExecStart=docker compose -f "/home/blockchain/tmp/docker-compose-mbase.yml" up --remove-orphans
+Restart=always
+TimeoutStopSec=infinity
+
+[Install]
+WantedBy=multi-user.target

lib/rudder/avro/block_specimen_decoder_encoder.ex

@@ -109,7 +109,7 @@ defmodule Rudder.Avro.BlockSpecimen do
         ) :: {:error, %{:__exception__ => true, :__struct__ => atom, optional(atom) => any}}
   def encode(result_json) do
     case Poison.decode(result_json) do
-      {:ok, result_map} -> Avrora.encode(result_map, schema_name: @schema_name, format: :ocf)
+      {:ok, result_map} -> Avrora.encode_plain(result_map, schema_name: @schema_name)
       err -> err
     end
   end

lib/rudder/proof_chain/block_specimen_event_listener.ex

@@ -21,6 +21,7 @@ defmodule Rudder.ProofChain.BlockSpecimenEventListener do
   def start() do
     reregister_process()
     Logger.info("starting event listener")
+    Application.ensure_all_started(:rudder)
     proofchain_address = Application.get_env(:rudder, :proofchain_address)
     push_bsps_to_process(Rudder.Journal.items_with_status(:discover))
     block_height = load_last_checked_block()

mix.exs

@@ -4,7 +4,7 @@ defmodule Rudder.MixProject do
   def project do
     [
       app: :rudder,
-      version: "0.2.2",
+      version: "0.2.6",
       elixir: "~> 1.14.3",
       start_permanent: Mix.env() == :prod,
       deps: deps()
@@ -26,6 +26,7 @@ defmodule Rudder.MixProject do
     [
       # system exec
       {:poison, "~> 5.0"},
+      {:distillery, "~> 2.0"},
 
       # frameworks
       {:broadway, "~> 1.0", override: true},
@@ -55,7 +56,7 @@ defmodule Rudder.MixProject do
       # networking
       {:certifi, "~> 2.9", override: true},
       {:cors_plug, "~> 3.0"},
-      {:finch, "~> 0.15.0"},
+      {:finch, "~> 0.16.0"},
       {:downstream, "~> 1.0"},
       {:websockex, "~> 0.4.3"},
       {:multipart, "~> 0.3.1"},
@@ -79,7 +80,6 @@ defmodule Rudder.MixProject do
       {:briefly, "~> 0.4.1"}
 
       # Unused
-
       # {:erlexec, "~> 2.0"},
       # {:etfs, path: "/Users/sudeep/repos/experiment/etfs"}
       # {:telemetry_metrics_prometheus, "~> 1.1"}