Merge branch 'unstable' into nh/node-22

ChainSafe · May 23, 2024 · 6fc54a0 · 6fc54a0
2 parents 90d1704 + 3121363
commit 6fc54a0
Show file tree

Hide file tree

Showing 130 changed files with 1,263 additions and 798 deletions.
diff --git a/.github/actions/setup-and-build/action.yml b/.github/actions/setup-and-build/action.yml
@@ -30,7 +30,7 @@ runs:
           lib/
           packages/*/lib
           packages/*/.git-data.json
-        key: ${{ runner.os }}-node-${{ inputs.node }}-${{ github.sha }}
+        key: ${{ runner.os }}-${{ runner.arch }}-node-${{ inputs.node }}-${{ github.sha }}
 
     - name: Install & build
       if: steps.cache-build-restore.outputs.cache-hit != 'true'
@@ -55,4 +55,4 @@ runs:
           lib/
           packages/*/lib
           packages/*/.git-data.json
-        key: ${{ runner.os }}-node-${{ inputs.node }}-${{ github.sha }}
+        key: ${{ runner.os }}-${{ runner.arch }}-node-${{ inputs.node }}-${{ github.sha }}
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -13,16 +13,10 @@ on:
       - stable
       - unstable
       - benchmark # For debugging
-    paths-ignore:
-      - 'dashboards/**'
-      - 'docs/**'
   pull_request:
     branches:
       - stable
       - unstable
-    paths-ignore:
-      - 'dashboards/**'
-      - 'docs/**'
   workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml
@@ -4,11 +4,7 @@ on:
   push:
     # We intentionally don't run push on feature branches. See PR for rational.
     branches: [unstable, stable]
-    paths-ignore:
-      - 'dashboards/**'
   pull_request:
-    paths-ignore:
-      - 'dashboards/**'
 
 jobs:
   build:

diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml
@@ -8,9 +8,6 @@ on:
   push:
     branches:
       - unstable
-    paths-ignore:
-      - 'dashboards/**'
-      - 'docs/**'
 
 jobs:
   npm:

diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml
@@ -9,13 +9,7 @@ on:
   push:
     # We intentionally don't run push on feature branches. See PR for rational.
     branches: [unstable, stable]
-    paths-ignore:
-      - 'dashboards/**'
-      - 'docs/**'
   pull_request:
-    paths-ignore:
-      - 'dashboards/**'
-      - 'docs/**'
   workflow_dispatch:
 
 env:

diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml
@@ -9,13 +9,7 @@ on:
   push:
     # We intentionally don't run push on feature branches. See PR for rational.
     branches: [unstable, stable]
-    paths-ignore:
-      - 'dashboards/**'
-      - 'docs/**'
   pull_request:
-    paths-ignore:
-      - 'dashboards/**'
-      - 'docs/**'
   workflow_dispatch:
     inputs:
       debug:

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -9,11 +9,7 @@ on:
   push:
     # We intentionally don't run push on feature branches. See PR for rational.
     branches: [unstable, stable]
-    paths-ignore:
-      - 'docs/**'
   pull_request:
-    paths-ignore:
-      - 'docs/**'
   workflow_dispatch:
 
 jobs:

diff --git a/.gitignore b/.gitignore
@@ -25,8 +25,10 @@ validators
 **/dist
 **/.nyc_output
 .tmp
-.vscode
 .npmrc
+.vscode/launch.json
+.vscode/settings.json
+.vscode/tasks.json
 
 # Tests artifacts
 packages/*/spec-tests*

diff --git a/.vscode/README.md b/.vscode/README.md
@@ -0,0 +1,3 @@
+Copy `launch.template.json` as `launch.json` and adapt to your needs.
+
+See [Debugging](../docs/pages/tools/debugging.md) for more details.
diff --git a/.vscode/launch.template.json b/.vscode/launch.template.json
@@ -0,0 +1,79 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "name": "Attach",
+      "port": 9229,
+      "request": "attach",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Beacon",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "smartStep": true,
+      "program": "${workspaceFolder}/packages/cli/bin/lodestar.js",
+      "args": [
+        "beacon"
+      ],
+      "console": "integratedTerminal"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Dev",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "smartStep": true,
+      "program": "${workspaceFolder}/packages/cli/bin/lodestar.js",
+      "args": [
+        "dev"
+      ],
+      "console": "integratedTerminal"
+    },
+    {
+      "name": "Test Current File",
+      "type": "node",
+      "request": "launch",
+      "program": "${workspaceFolder}/node_modules/.bin/vitest",
+      "args": [
+        "--run",
+        "${file}",
+        "-t",
+        "${input:testName}",
+        "--pool",
+        "threads",
+        "--poolOptions.threads.singleThread"
+      ],
+      "cwd": "${workspaceFolder}/${input:packageName}",
+      "console": "integratedTerminal",
+      "skipFiles": [
+        "<node_internals>/**"
+      ]
+    }
+  ],
+  "inputs": [
+    {
+      "id": "packageName",
+      "type": "command",
+      "command": "extension.commandvariable.transform",
+      "args": {
+        "text": "${relativeFileDirname}",
+        "find": "^(packages/[^/]+).*",
+        "replace": "$1"
+      }
+    },
+    {
+      "id": "testName",
+      "type": "promptString",
+      "description": "Enter the test name to run, leave empty to run all"
+    }
+  ]
+}
diff --git a/.wordlist.txt b/.wordlist.txt
@@ -41,6 +41,7 @@ Geth
 Github
 Gossipsub
 Grafana
+Grandine
 HTTPS
 HackMD
 Homebrew
@@ -60,6 +61,7 @@ LTS
 Lerna
 MEV
 MacOS
+Metamask
 Monorepo
 NPM
 NVM
@@ -129,6 +131,7 @@ enum
 env
 envs
 ephemery
+ethers
 flamegraph
 flamegraphs
 getNetworkIdentity

diff --git a/docs/pages/contribution/testing/index.md b/docs/pages/contribution/testing/index.md
@@ -2,6 +2,12 @@
 
 Testing is critical to the Lodestar project and there are many types of tests that are run to build a product that is both effective AND efficient. This page will help to break down the different types of tests you will find in the Lodestar repo.
 
+There are few flags you can set through env variables to override behavior of testing and it's output.
+
+| ENV variable      | Effect | Impact                                                                                                      |
+| ----------------- | ------ | ----------------------------------------------------------------------------------------------------------- |
+| TEST_COMPACT_DIFF | All    | Will strip down the object difference rendered during test failures. Very useful for large object matching. |
+
 ### Unit Tests
 
 This is the most fundamental type of test in most code bases. In all instances mocks, stubs and other forms of isolation are used to test code on a functional, unit level. See the [Unit Tests](./unit-tests.md) page for more information.

diff --git a/docs/pages/faqs.md b/docs/pages/faqs.md
@@ -4,6 +4,19 @@ This section of the documentation will cover common questions and encounters oft
 
 ## Troubleshooting Lodestar
 
+### Running a beacon node
+
+:::note "Heap memory limit"
+Lodestar beacon node requires at least 8GB of heap space. While the `lodestar` script and the official docker image correctly sets the appropriate value, it might be necessary to manually set it for some specific scenario.
+
+The simplest way to achieve this is via the `NODE_OPTIONS` environment variable or by passing [`--max-old-space-size`](https://nodejs.org/api/cli.html#--max-old-space-sizesize-in-megabytes) directly to the node binary
+
+```bash
+NODE_OPTIONS: --max-old-space-size=8192
+```
+
+:::
+
 ### Using Kubernetes
 
 :::note "Unknown arguments error"

diff --git a/docs/pages/introduction.md b/docs/pages/introduction.md
@@ -15,12 +15,13 @@ In an effort to promote client diversity there are several beacon-nodes being de
 - [Lighthouse](https://lighthouse.sigmaprime.io/)
 - [Teku](https://consensys.net/knowledge-base/ethereum-2/teku/)
 - [Nimbus](https://nimbus.team/)
+- [Grandine](https://grandine.io)
 
 ## Why Client Diversity?
 
 The Ethereum network's robustness is significantly enhanced by its client diversity, whereby multiple, independently-developed clients conforming to a common specification facilitate seamless interaction and function equivalently across nodes. This client variety not only fosters a rich ecosystem but also provides a buffer against network-wide issues stemming from bugs or malicious attacks targeted at particular clients. For instance, during the Shanghai denial-of-service attack in 2016, the diversified client structure enabled the network to withstand the assault, underscoring the resilience afforded by multiple client configurations.
 
-On the consensus layer, client distribution is crucial for maintaining network integrity and finality, ensuring transactions are irreversible once validated. A balanced spread of nodes across various clients helps mitigate risks associated with potential bugs or attacks that could, in extreme cases, derail the consensus process or lead to incorrect chain splits, thereby jeopardizing the network's stability and trust. While the data suggests a dominance of Prysm client on the consensus layer, efforts are ongoing to promote a more even distribution among others like Lighthouse, Teku, and Nimbus. Encouraging the adoption of minority clients, bolstering their documentation, and leveraging real-time client diversity dashboards are among the strategies being employed to enhance client diversity, which in turn fortifies the Ethereum consensus layer against adversities and fosters a healthier decentralized network ecosystem.
+On the consensus layer, client distribution is crucial for maintaining network integrity and finality, ensuring transactions are irreversible once validated. A balanced spread of nodes across various clients helps mitigate risks associated with potential bugs or attacks that could, in extreme cases, derail the consensus process or lead to incorrect chain splits, thereby jeopardizing the network's stability and trust. While the data suggests a dominance of Prysm client on the consensus layer, efforts are ongoing to promote a more even distribution among others like Lighthouse, Teku, Nimbus and Grandine. Encouraging the adoption of minority clients, bolstering their documentation, and leveraging real-time client diversity dashboards are among the strategies being employed to enhance client diversity, which in turn fortifies the Ethereum consensus layer against adversities and fosters a healthier decentralized network ecosystem.
 
 The non-finality event in May 2023 on the Ethereum network posed a significant challenge. The issue arose from attestations for a fork, which necessitated state replays to validate the attestations, causing a notable strain on system resources. As a result, nodes fell out of sync, which deterred the accurate tracking of the actual head of the chain. This situation was exacerbated by a decline in attestations during specific epochs, further hampering the consensus mechanism. The Lodestar team noticed late attestations several weeks prior to the event and implemented a feature that attempted to address such challenges by not processing untimely attestations, and thus not requiring expensive state replays. While it was done for slightly different reasons, the result was the same. Lodestar was able to follow the chain correctly and helped to stabilize the network. This example underscored the importance of client diversity and network resilience against potential forks and replay attacks. These are considered realistic threats, especially in the context of system complexity like in Ethereum's consensus mechanism.
 

diff --git a/docs/pages/tools/debugging.md b/docs/pages/tools/debugging.md
@@ -0,0 +1,24 @@
+# Debugging
+
+This page describes different approaches for debugging Lodestar.
+
+## VS Code launch config
+
+The simplest way to debug is to use the provided [launch.template.json](https://github.com/ChainSafe/lodestar/blob/unstable/.vscode/launch.template.json) `configurations`. Copy them as `.vscode/launch.json` and they will be made available in the `Run and Debug` section in VS Code. Adapt as needed, e.g. by adding additional arguments to the beacon [configuration](https://github.com/ChainSafe/lodestar/blob/unstable/.vscode/launch.template.json#L22) to match your needs.
+
+VS Code supports debugging Workers out of the box when using those configurations.
+
+## Attach to running process
+
+Remote `lodestar` processes can also be debugged by leveraging [node:inspector](https://nodejs.org/api/inspector.html). Adding `--inspect` to the node CLI (e.g. `NODE_OPTIONS=--inspect ./lodestar beacon`) allows to debug the main thread. To debug a specific `Worker`, follow those steps:
+
+- remove `--inspect` from `node` CLI
+- add following code to the `worker`
+
+```js
+import inspector from "node:inspector";
+inspector.open();
+inspector.waitForDebugger();
+```
+
+Use VS Code or Chrome devtools to debug those processes.
diff --git a/docs/sidebars.ts b/docs/sidebars.ts
@@ -4,6 +4,11 @@ const sidebars: SidebarsConfig = {
   tutorialSidebar: [
     "index",
     "introduction",
+    {
+      type: "doc",
+      label: "Security",
+      id: "security",
+    },
     {
       type: "category",
       label: "Getting Started",
@@ -37,7 +42,19 @@ const sidebars: SidebarsConfig = {
     {
       type: "category",
       label: "Light Client and Prover",
-      items: ["lightclient-prover/lightclient-cli", "lightclient-prover/lightclient", "lightclient-prover/prover"],
+      items: [
+        "lightclient-prover/lightclient-cli",
+        {
+          type: "doc",
+          label: "Light Client",
+          id: "lightclient-prover/lightclient",
+        },
+        {
+          type: "doc",
+          label: "Prover",
+          id: "lightclient-prover/prover",
+        },
+      ],
     },
     {
       type: "category",
@@ -49,7 +66,17 @@ const sidebars: SidebarsConfig = {
       type: "category",
       label: "Contributing",
       items: [
+        {
+          type: "doc",
+          label: "Getting Started",
+          id: "contribution/getting-started",
+        },
         "contribution/depgraph",
+        {
+          type: "doc",
+          label: "Dev CLI Reference",
+          id: "contribution/dev-cli",
+        },
         {
           type: "category",
           label: "Testing",
@@ -67,7 +94,7 @@ const sidebars: SidebarsConfig = {
     {
       type: "category",
       label: "Tools",
-      items: ["tools/flamegraphs", "tools/heap-dumps", "tools/core-dumps"],
+      items: ["tools/debugging", "tools/flamegraphs", "tools/heap-dumps", "tools/core-dumps"],
     },
     {
       type: "category",

diff --git a/lerna.json b/lerna.json
@@ -4,7 +4,7 @@
   ],
   "npmClient": "yarn",
   "useNx": true,
-  "version": "1.18.0",
+  "version": "1.18.1",
   "stream": true,
   "command": {
     "version": {

diff --git a/package.json b/package.json
@@ -51,8 +51,8 @@
     "@types/node": "^20.12.8",
     "@typescript-eslint/eslint-plugin": "^7.2.0",
     "@typescript-eslint/parser": "^7.2.0",
-    "@vitest/coverage-v8": "^1.5.3",
-    "@vitest/browser": "^1.5.3",
+    "@vitest/browser": "^1.6.0",
+    "@vitest/coverage-v8": "^1.6.0",
     "crypto-browserify": "^3.12.0",
     "dotenv": "^16.4.5",
     "electron": "^26.2.2",
@@ -78,11 +78,11 @@
     "ts-node": "^10.9.2",
     "typescript": "^5.4.2",
     "typescript-docs-verifier": "^2.5.0",
+    "vite": "^5.2.11",
+    "vite-plugin-dts": "^3.9.1",
     "vite-plugin-node-polyfills": "^0.21.0",
     "vite-plugin-top-level-await": "^1.4.1",
-    "vite-plugin-dts": "^3.8.3",
-    "vite": "^5.2.10",
-    "vitest": "^1.5.3",
+    "vitest": "^1.6.0",
     "vitest-when": "^0.3.1",
     "wait-port": "^1.1.0",
     "webdriverio": "^8.36.1"
@@ -91,7 +91,7 @@
     "@puppeteer/browsers": "^2.1.0",
     "dns-over-http-resolver": "^2.1.1",
     "loupe": "^2.3.6",
-    "vite": "^5.2.10",
+    "vite": "^5.2.11",
     "testcontainers/**/nan": "^2.19.0"
   }
 }