Merge pull request #94 from skalenetwork/develop

Develop to beta

.github/workflows/comprehensive.yml

@@ -276,27 +276,6 @@ jobs:
         cd comprehensive-test
         cd engine 
         yarn install
-        cd ../..
-
-    - name: SELF-TEST A - use IMA to browse S-Chain via node 00-00 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.1:2164 || true
-
-    - name: SELF-TEST B - use IMA to browse S-Chain via node 00-01 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.2:2264 || true
-
-    - name: SELF-TEST C - use IMA to browse S-Chain via node 01-00 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.3:2364 || true
-
-    - name: SELF-TEST D - use IMA to browse S-Chain via node 01-01 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.4:2464 || true
 
     - name: INIT - download SGX Wallet
       working-directory: ${{env.working-directory}}

.github/workflows/container-test.yml

@@ -286,27 +286,6 @@ jobs:
         cd comprehensive-test
         cd engine 
         yarn install
-        cd ../..
-
-    - name: SELF-TEST A - use IMA to browse S-Chain via node 00-00 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.1:2164 || true
-
-    - name: SELF-TEST B - use IMA to browse S-Chain via node 00-01 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.2:2264 || true
-
-    - name: SELF-TEST C - use IMA to browse S-Chain via node 01-00 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.3:2364 || true
-
-    - name: SELF-TEST D - use IMA to browse S-Chain via node 01-01 and test last is alive
-      working-directory: ${{env.working-directory}}
-      run: |
-        node ./src/build/main.js --colors --no-gathered --browse-s-chain --url-s-chain=http://127.0.0.4:2464 || true
 
     - name: INIT - build IMA docker container
       working-directory: ${{env.working-directory}}

.github/workflows/network-browser-test.yml

@@ -1,5 +1,5 @@
 name: network-browser test
-on: [push, pull_request]
+on: [push]
 env:
   ETH_PRIVATE_KEY: ${{ secrets.ETH_PRIVATE_KEY }}
   MANAGER_TAG: "1.9.3-beta.0"
@@ -25,4 +25,4 @@ jobs:
 
       - name: Run network-browser tests
         working-directory: network-browser
-        run: bash run_tests.sh
+        run: bash run_tests.sh

.github/workflows/publish.yml

@@ -69,6 +69,11 @@ jobs:
       with:
         python-version: 3.8
 
+    - name: Ensure deps are latest and build everything
+      run: |
+        yarn install
+        yarn rebuild
+
     - name: Build and publish container
       run: |
         export BRANCH=${GITHUB_REF##*/}

CODEOWNERS

@@ -1,2 +1,2 @@
-* @sergiy-skalelabs @kladkogex @DmytroNazarenko
+* @sergiy-skalelabs @dmytrotkk @DmytroNazarenko
 *.md @skalenetwork/docowners

Dockerfile

@@ -43,6 +43,7 @@ COPY package.json package.json
 
 COPY runner runner
 COPY src src
+COPY src/pow src/build/pow
 RUN mkdir IMA
 COPY IMA/proxy IMA/proxy
 COPY IMA/package.json IMA/package.json

README.md

@@ -18,9 +18,10 @@ A module implementing core IMA functionality.
 
 Data validity verifier module. See [OWASP document](https://www.gitbook.com/download/pdf/book/checkmarx/JS-SCP).
 
-#### SKALE Observer
+#### SKALE Network Browser
 
-[SKALE Network Browser](src/SNB.md). Responsible for providing description of all SKALE chains.
+Component responsible for providing description of all SKALE chains. See detailed spec:
+[SKALE Network Browser](docs/SNB.md)
 
 #### IMA Log
 

docs/SNB.md

@@ -0,0 +1,156 @@
+# SKALE Network Browser
+
+## General Description
+
+**SKALE Network Browser** (**SNB**) is part of IMA responsible for collecting data about all SKALE chains from skale-manager.
+SKALE Network Browser can work as a standalone tool to browse SKALE network or work as a part of IMA agent.
+
+- SNB is implemented as a separate module of IMA agent.
+- SNB can work as a standalone tool to browse SKALE network or work as a part of IMA agent.
+- SNB saves results of SKALE network scan to the specified file.
+- SNB runs in an infinite loop and periodically refreshes SKALE network data.
+- SNB runs as a separate process and managed by `startup.js` script.
+
+## Architecture
+
+### High-Level Architecture
+
+SNB operate as an independent process within the IMA agent container, interacting with SKALE Manager and sChains to gather network information about SKALE Chains as connections between them.
+
+In the latest version, SNB is implemented as a separate module of IMA agent and managed by `startup.js` script (see Diagram 1).
+
+### Implementation Details
+
+SNB runs in an endless loop - `safeNetworkBrowserLoop` and periodically refreshes SKALE network data (see Diagram 2).
+
+It has multiple delays/timeouts that serve different purposes:
+
+-   `POST_ERROR_DELAY` - delay before retry if error happened in browser loop (seconds, default: `5`)
+-   `NETWORK_BROWSER_DELAY` - delay between iterations of the network-browser (seconds, default: `10800`)
+-   `NETWORK_BROWSER_TIMEOUT` - maximum amount of time allocated to the browse function (seconds, default: `1200`)
+
+See more details in the [network-browser README](../network-browser/README.md).
+
+- Mainnet provider is created using `MAINNET_RPC_URL` env variable and optional `MULTICALL` env variable
+- `sChainsInternal` and `nodes` contracts are created using `MANAGER_ABI_PATH` env variable
+- `SCHAIN_URL` pinged in a loop to determine if local/remote skaled is running and ready to accept requests
+
+After this, SNB starts its main loop.
+On each iteration, SNB performs the following steps:
+
+- All sChain hashes are requested from `sChainsInternal` contract
+- For each sChain, its data is requested from `sChainsInternal` contract using `schains` call
+- Optional step: if `CONNECTED_ONLY` env variable is set to `true`, only connected sChains are processed
+- To determine if sChain is connected, sChain provider is inited using `SCHAIN_RPC_URL` env variable and `isConnectedChain` call is performed on `messageProxy` contract on local sChain
+- For each chain nodes in the group are requested from `sChainsInternal` contract using `getNodesInGroup` call
+- For each node id node data is requested from `nodes` contract using `nodes` call, `getNodeDomainName` call and `getSchainHashesForNode` call
+- Endpoints are computed using `nodeStruct` function for each node
+- Resulting data is saved to the file specified in `IMA_NETWORK_BROWSER_DATA_PATH` env variable along with the timestamp
+
+All calls in the loops are performed using `multicall` provider if `MULTICALL` env variable is set to `true`.
+See Diagram 3 for more details.
+
+## Component Diagrams
+
+### Diagram 1. High-Level Architecture
+This diagram shows how IMA container is structured and how SNB is integrated into it:
+
+![IMA Container Structure](./ima-container-structure.png)
+
+### Diagram 2. SNB Loop
+This diagram shows SNB’s browse loop:
+
+![SNB Loop](./snb-loop.png)
+
+### Diagram 3. SNB Internal Structure
+This diagram shows SNB’s internal structure:
+
+![SNB Structure](./snb-structure.png)
+
+## Data format
+
+SNB stores data in JSON format. The data is stored in the following format:
+
+```json
+{
+  "updatedAt": 1703247331,
+  "schains": [
+    {
+      "name": "chain-name",
+      "mainnetOwner": "0x5cF21aeaAAA5472b9b11AE3c73A392466EeF854e",
+      "indexInOwnerList": "0",
+      "partOfNode": "16",
+      "lifetime": "43200",
+      "startDate": "1654778157",
+      "startBlock": "14932629",
+      "deposit": "0",
+      "index": "26",
+      "generation": "1",
+      "originator": "0x4f7e825E1C0a42924016115897e39c9d93D04aD1",
+      "chainId": 1026062157,
+      "nodes": [
+        {
+          "name": "node-name",
+          "ip": "21.83.36.61",
+          "publicIP": "21.33.36.60",
+          "port": "10000",
+          "startBlock": "10966891",
+          "lastRewardDate": "1700957519",
+          "finishTime": "0",
+          "status": "3",
+          "validatorId": "52",
+          "domainName": "test.domain.com",
+          "schainHashes": [
+            "0x0000073273d9f3c4e488771393160391885dfa507a20fb697be411ad3411d363",
+            "0x000003d91725cdc8aab2c5aeef5e5dbee73162670abe6353e31ac4348e616d0e",
+            "0x0000017723ca21de167b6735af8638d9f1d82345ec97d288f4d6918662f0a865",
+            "0x000003d28774d2845ee8b9f656cc77328199a7a69b03ce2b9578f230be679c9f"
+          ],
+          "endpoints": {
+            "ports": {
+              "http": 10195,
+              "https": 10200,
+              "ws": 10194,
+              "wss": 10199,
+              "infoHttp": 10201
+            },
+            "domain": {
+              "http": "http://test.domain.com:10195",
+              "https": "https://test.domain.com:10200",
+              "ws": "ws://test.domain.com:10194",
+              "wss": "wss://test.domain.com:10199",
+              "infoHttp": "http://test.domain.com:10201"
+            },
+            "ip": {
+              "http": "http://11.23.36.60:10195",
+              "https": "https://21.23.36.60:10200",
+              "ws": "ws://11.83.26.60:10194",
+              "wss": "wss://11.22.26.60:10199",
+              "infoHttp": "http://11.22.36.60:10201"
+            }
+          }
+        }
+      ]
+    }
+  ]
+}
+```
+
+In may contain either the entire SKALE network or connected chains only, depending on the configuration.
+
+## Usage
+
+SNB usage is fully covered in the [network-browser README](../network-browser/README.md), including required and optional environment variables, development and production modes, etc.
+
+## Testing
+
+SNB is covered by unit tests and also tested as a part of integration tests of ima-agent.
+
+Unit tests in the `network-browser` works as follows:
+
+- Local hardhat network is started locally
+- `skale-manager` contracts are deployed to the local network
+- Before each test, a set of nodes and sChains are created on `skale-manager`
+- Tests are executed using `bun:test` framework
+
+Units tests are located in the `network-browser/tests` folder.

network-browser/index.ts

@@ -26,7 +26,7 @@ import {
     getMainnetManagerAbi,
     getMainnetProvider
 } from './src/contracts'
-import { delay, pingUrl, withTimeout } from './src/tools'
+import { delay, getLoggerConfig, checkEndpoint, withTimeout } from './src/tools'
 import { BrowserTimeoutError } from './src/errors'
 import { browse } from './src/browser'
 import {
@@ -37,14 +37,12 @@ import {
     NETWORK_BROWSER_DELAY,
     MULTICALL,
     CONNECTED_ONLY,
-    SCHAIN_RPC_URL,
-    LOG_LEVEL,
-    LOG_PRETTY
+    SCHAIN_RPC_URL
 } from './src/constants'
 
 import { Logger, type ILogObj } from 'tslog'
 
-const log = new Logger<ILogObj>({ minLevel: LOG_LEVEL, stylePrettyLogs: LOG_PRETTY })
+const log = new Logger<ILogObj>(getLoggerConfig('loop'))
 
 async function safeNetworkBrowserLoop() {
     log.info(`Running network-browser...`)
@@ -55,24 +53,28 @@ async function safeNetworkBrowserLoop() {
     log.info(`NETWORK_BROWSER_TIMEOUT: ${NETWORK_BROWSER_TIMEOUT}`)
     log.info(`NETWORK_BROWSER_DELAY: ${NETWORK_BROWSER_DELAY}`)
 
+    log.info(`Trying to connect to the sChain RPC: ${SCHAIN_RPC_URL}`)
+    await checkEndpoint(SCHAIN_RPC_URL)
+    log.info(`Trying to connect to the mainnet RPC`)
+    await checkEndpoint(MAINNET_RPC_URL)
+
     const provider = await getMainnetProvider(MAINNET_RPC_URL, MULTICALL)
     const managerAbi = getMainnetManagerAbi()
     const schainsInternal = schainsInternalContract(managerAbi, provider)
     const nodes = nodesContract(managerAbi, provider)
-    log.info(`Trying to connect to the sChain RPC: ${SCHAIN_RPC_URL}`)
-    await pingUrl(SCHAIN_RPC_URL)
+
     while (true) {
         try {
             await withTimeout(browse(schainsInternal, nodes), NETWORK_BROWSER_TIMEOUT)
             await delay(NETWORK_BROWSER_DELAY)
         } catch (error) {
             if (error instanceof BrowserTimeoutError) {
-                console.error(
+                log.error(
                     `A timeout (${NETWORK_BROWSER_TIMEOUT} ms) error occurred:`,
                     error.message
                 )
             } else {
-                console.error('An error occurred in browse:', error)
+                log.error('An error occurred in browse:', error)
             }
             await delay(POST_ERROR_DELAY)
         }

network-browser/src/browser.ts

@@ -33,10 +33,10 @@ import {
     filterConnectedHashes
 } from './schains'
 import { getNodesGroups } from './nodes'
-import { CONNECTED_ONLY, IMA_NETWORK_BROWSER_DATA_PATH, LOG_LEVEL, LOG_PRETTY } from './constants'
-import { writeJson, currentTimestamp, chainIdInt } from './tools'
+import { CONNECTED_ONLY, IMA_NETWORK_BROWSER_DATA_PATH } from './constants'
+import { writeJson, currentTimestamp, chainIdInt, getLoggerConfig } from './tools'
 
-const log = new Logger<ILogObj>({ minLevel: LOG_LEVEL, stylePrettyLogs: LOG_PRETTY })
+const log = new Logger<ILogObj>(getLoggerConfig('browser'))
 
 export async function browse(schainsInternal: Contract, nodes: Contract): Promise<void> {
     log.info('Browse iteration started, collecting chains')

network-browser/src/constants.ts

@@ -56,3 +56,6 @@ export const NETWORK_BROWSER_TIMEOUT = secondsEnv(process.env.NETWORK_BROWSER_TI
 
 export const LOG_LEVEL = optionalEnvNumber('NETWORK_BROWSER_LOG_LEVEL', 1)
 export const LOG_PRETTY = booleanEnv('NETWORK_BROWSER_LOG_PRETTY', false)
+export const LOG_FORMAT =
+    '{{yyyy}}.{{mm}}.{{dd}} {{hh}}:{{MM}}:{{ss}}:{{ms}}\t{{logLevelName}}\t{{name}}\t'
+export const LOG_PREFIX = 'snb::'